SteNicholas commented on code in PR #1469:
URL: https://github.com/apache/paimon-rust/pull/1469#discussion_r4656859051
##########
crates/paimon/src/arrow/format/parquet.rs:
##########
@@ -2807,51 +2834,96 @@ fn build_predicate_page_selection(
});
}
- Ok(combined)
+ combined
}
-/// Collect the file-field indices referenced by leaf predicates (recursing
-/// through compound nodes). Indices are already file-level (see caller).
-fn collect_leaf_field_indices(predicate: &Predicate, out: &mut Vec<usize>) {
- match predicate {
- Predicate::NestedLeaf { index, .. } => out.push(*index),
- Predicate::Leaf { index, .. } => out.push(*index),
- Predicate::And(children) | Predicate::Or(children) => {
- for child in children {
- collect_leaf_field_indices(child, out);
- }
- }
- Predicate::Not(child) => collect_leaf_field_indices(child, out),
- Predicate::AlwaysTrue | Predicate::AlwaysFalse => {}
- }
+fn offset_index_valid(metadata: &ParquetMetaData, file_size: u64) -> bool {
+ let Some(groups) = metadata.offset_index() else {
+ return true;
+ };
+ groups.len() == metadata.num_row_groups()
+ && groups.iter().enumerate().all(|(group_index, offsets)| {
+ let group = metadata.row_group(group_index);
+ let stats = metadata
+ .column_index()
+ .and_then(|index| index.get(group_index));
+ offsets.len() == group.num_columns()
+ && offsets.iter().enumerate().all(|(column_index, offset)| {
+ page_locations_valid(
+ offset.page_locations(),
+ group.num_rows(),
+ group.column(column_index),
+ file_size,
+ stats.and_then(|columns| columns.get(column_index)),
+ )
+ })
+ })
}
-/// Validate that an OffsetIndex's page row boundaries are well-formed for a
row
-/// group of `rg_rows` rows: the first page starts at row 0, `first_row_index`
-/// is strictly increasing, and every value stays within `[0, rg_rows]`.
-///
-/// Malformed metadata (negative, non-monotonic, or out-of-range boundaries)
-/// would otherwise produce invalid row ranges — huge values from `i64 as
usize`
-/// underflow, or ranges past the row group — that panic
-/// `RowSelection::from_consecutive_ranges`. Callers fail open when this
returns
-/// `false`.
-fn page_boundaries_valid(
+fn page_locations_valid(
pages: &[parquet::file::page_index::offset_index::PageLocation],
- rg_rows: usize,
+ row_count: i64,
+ column: &ColumnChunkMetaData,
+ file_size: u64,
+ column_index: Option<&ColumnIndexMetaData>,
) -> bool {
- let rg_rows = rg_rows as i64;
- let mut prev: i64 = -1;
- for (page_idx, page) in pages.iter().enumerate() {
- let first_row = page.first_row_index;
- if page_idx == 0 && first_row != 0 {
- return false;
- }
- if first_row <= prev || first_row > rg_rows {
- return false;
- }
- prev = first_row;
+ let column_index = column_index.filter(|index| !matches!(index,
ColumnIndexMetaData::NONE));
+ if column_index.is_some_and(|index| index.num_pages() != pages.len() as
u64) {
+ return false;
+ }
+ let Some(first) = pages.first() else {
+ return row_count == 0;
+ };
+ let data_start = column.data_page_offset();
+ let chunk_start = column.dictionary_page_offset().unwrap_or(data_start);
+ let Some(chunk_end) = chunk_start.checked_add(column.compressed_size())
else {
+ return false;
+ };
+ if !(0..=data_start).contains(&chunk_start)
+ || !u64::try_from(chunk_end).is_ok_and(|end| end <= file_size)
+ || first.first_row_index != 0
+ || first.offset != data_start
+ {
+ return false;
+ }
+ // Arrow reads each indexed page by its recorded size, including the
header.
+ // A gap can truncate a projected page even when another file supplies the
filter.
+ pages.iter().enumerate().all(|(index, page)| {
+ let (next_row, next_byte) = pages.get(index + 1).map_or((row_count,
chunk_end), |next| {
+ (next.first_row_index, next.offset)
+ });
+ page.first_row_index < next_row
+ && page.compressed_page_size > 0
+ && page
+ .offset
+ .checked_add(i64::from(page.compressed_page_size))
+ .is_some_and(|end| end == next_byte)
+ && column_index.is_none_or(|stats| {
Review Comment:
Fixed in
[b6d10803](https://github.com/apache/paimon-rust/commit/b6d10803a153c667a92434148659e054607f95a9).
Merged pruning now requires independently checkable page row counts. When they
are unavailable, it removes the provider's optional OffsetIndex and decodes
whole row-group column chunks before selecting rows. Projected providers are
also checked when the candidate range equals the caller's existing sparse
selection.
The regression uses 37-row `id` pages and one-row `rating` pages, then
changes the second `id` page's first row from 37 to 36. It covers both an
OffsetIndex-only file and a ColumnIndex without row-count histograms. The
result stays `(id=36, _ROW_ID=100036)`, including when row 100036 is already
selected. [Regression
test](https://github.com/apache/paimon-rust/blob/b6d10803a153c667a92434148659e054607f95a9/crates/paimon/src/table/data_evolution_reader/candidate_ranges_tests.rs#L793)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]