SteNicholas commented on code in PR #1469:
URL: https://github.com/apache/paimon-rust/pull/1469#discussion_r4653626635
##########
crates/paimon/src/arrow/format/parquet.rs:
##########
@@ -2807,51 +2827,41 @@ fn build_predicate_page_selection(
});
}
- Ok(combined)
-}
-
-/// Collect the file-field indices referenced by leaf predicates (recursing
-/// through compound nodes). Indices are already file-level (see caller).
-fn collect_leaf_field_indices(predicate: &Predicate, out: &mut Vec<usize>) {
- match predicate {
- Predicate::NestedLeaf { index, .. } => out.push(*index),
- Predicate::Leaf { index, .. } => out.push(*index),
- Predicate::And(children) | Predicate::Or(children) => {
- for child in children {
- collect_leaf_field_indices(child, out);
- }
- }
- Predicate::Not(child) => collect_leaf_field_indices(child, out),
- Predicate::AlwaysTrue | Predicate::AlwaysFalse => {}
- }
+ combined
}
-/// Validate that an OffsetIndex's page row boundaries are well-formed for a
row
-/// group of `rg_rows` rows: the first page starts at row 0, `first_row_index`
-/// is strictly increasing, and every value stays within `[0, rg_rows]`.
-///
-/// Malformed metadata (negative, non-monotonic, or out-of-range boundaries)
-/// would otherwise produce invalid row ranges — huge values from `i64 as
usize`
-/// underflow, or ranges past the row group — that panic
-/// `RowSelection::from_consecutive_ranges`. Callers fail open when this
returns
-/// `false`.
-fn page_boundaries_valid(
+fn page_locations_valid(
pages: &[parquet::file::page_index::offset_index::PageLocation],
- rg_rows: usize,
+ row_count: i64,
+ column: &ColumnChunkMetaData,
+ file_size: u64,
) -> bool {
- let rg_rows = rg_rows as i64;
- let mut prev: i64 = -1;
- for (page_idx, page) in pages.iter().enumerate() {
- let first_row = page.first_row_index;
- if page_idx == 0 && first_row != 0 {
- return false;
- }
- if first_row <= prev || first_row > rg_rows {
- return false;
- }
- prev = first_row;
+ let Some(first) = pages.first() else {
+ return row_count == 0;
+ };
+ let data_start = column.data_page_offset();
+ let chunk_start = column.dictionary_page_offset().unwrap_or(data_start);
+ let Some(chunk_end) = chunk_start.checked_add(column.compressed_size())
else {
+ return false;
+ };
+ if !(0..=data_start).contains(&chunk_start)
+ || !u64::try_from(chunk_end).is_ok_and(|end| end <= file_size)
+ || first.first_row_index != 0
+ || first.offset != data_start
+ {
+ return false;
}
- true
+ pages.iter().enumerate().all(|(index, page)| {
+ let (next_row, next_byte) = pages.get(index + 1).map_or((row_count,
chunk_end), |next| {
+ (next.first_row_index, next.offset)
+ });
+ page.first_row_index < next_row
+ && page.compressed_page_size > 0
+ && page
+ .offset
+ .checked_add(i64::from(page.compressed_page_size))
+ .is_some_and(|end| end <= next_byte)
Review Comment:
Fixed in
[789b8d6c](https://github.com/apache/paimon-rust/commit/789b8d6c3523a5e2baf594b6c7e4fd11d9a76cc9).
Each indexed data page must end where the next begins, and the last must end
at the column-chunk boundary. Gaps or overlaps discard the optional OffsetIndex
and fall back to whole row-group column chunks.
The
[regression](https://github.com/apache/paimon-rust/blob/789b8d6c3523a5e2baf594b6c7e4fd11d9a76cc9/crates/paimon/src/table/data_evolution_reader/candidate_ranges_tests.rs#L869)
shortens the last page's recorded size by one byte while leaving the data
intact, for both projected and filtered files. `rating = 4095` returns the
expected row with pushdown enabled and disabled, including when the caller
selects only the last two rows. The same test covers oversized entries,
first-page truncation, gaps and overlaps.
##########
crates/paimon/src/arrow/format/parquet.rs:
##########
@@ -2807,51 +2834,96 @@ fn build_predicate_page_selection(
});
}
- Ok(combined)
+ combined
}
-/// Collect the file-field indices referenced by leaf predicates (recursing
-/// through compound nodes). Indices are already file-level (see caller).
-fn collect_leaf_field_indices(predicate: &Predicate, out: &mut Vec<usize>) {
- match predicate {
- Predicate::NestedLeaf { index, .. } => out.push(*index),
- Predicate::Leaf { index, .. } => out.push(*index),
- Predicate::And(children) | Predicate::Or(children) => {
- for child in children {
- collect_leaf_field_indices(child, out);
- }
- }
- Predicate::Not(child) => collect_leaf_field_indices(child, out),
- Predicate::AlwaysTrue | Predicate::AlwaysFalse => {}
- }
+fn offset_index_valid(metadata: &ParquetMetaData, file_size: u64) -> bool {
+ let Some(groups) = metadata.offset_index() else {
+ return true;
+ };
+ groups.len() == metadata.num_row_groups()
+ && groups.iter().enumerate().all(|(group_index, offsets)| {
+ let group = metadata.row_group(group_index);
+ let stats = metadata
+ .column_index()
+ .and_then(|index| index.get(group_index));
+ offsets.len() == group.num_columns()
+ && offsets.iter().enumerate().all(|(column_index, offset)| {
+ page_locations_valid(
+ offset.page_locations(),
+ group.num_rows(),
+ group.column(column_index),
+ file_size,
+ stats.and_then(|columns| columns.get(column_index)),
+ )
+ })
+ })
}
-/// Validate that an OffsetIndex's page row boundaries are well-formed for a
row
-/// group of `rg_rows` rows: the first page starts at row 0, `first_row_index`
-/// is strictly increasing, and every value stays within `[0, rg_rows]`.
-///
-/// Malformed metadata (negative, non-monotonic, or out-of-range boundaries)
-/// would otherwise produce invalid row ranges — huge values from `i64 as
usize`
-/// underflow, or ranges past the row group — that panic
-/// `RowSelection::from_consecutive_ranges`. Callers fail open when this
returns
-/// `false`.
-fn page_boundaries_valid(
+fn page_locations_valid(
pages: &[parquet::file::page_index::offset_index::PageLocation],
- rg_rows: usize,
+ row_count: i64,
+ column: &ColumnChunkMetaData,
+ file_size: u64,
+ column_index: Option<&ColumnIndexMetaData>,
) -> bool {
- let rg_rows = rg_rows as i64;
- let mut prev: i64 = -1;
- for (page_idx, page) in pages.iter().enumerate() {
- let first_row = page.first_row_index;
- if page_idx == 0 && first_row != 0 {
- return false;
- }
- if first_row <= prev || first_row > rg_rows {
- return false;
- }
- prev = first_row;
+ let column_index = column_index.filter(|index| !matches!(index,
ColumnIndexMetaData::NONE));
+ if column_index.is_some_and(|index| index.num_pages() != pages.len() as
u64) {
+ return false;
+ }
+ let Some(first) = pages.first() else {
+ return row_count == 0;
+ };
+ let data_start = column.data_page_offset();
+ let chunk_start = column.dictionary_page_offset().unwrap_or(data_start);
+ let Some(chunk_end) = chunk_start.checked_add(column.compressed_size())
else {
+ return false;
+ };
+ if !(0..=data_start).contains(&chunk_start)
+ || !u64::try_from(chunk_end).is_ok_and(|end| end <= file_size)
+ || first.first_row_index != 0
+ || first.offset != data_start
+ {
+ return false;
+ }
+ // Arrow reads each indexed page by its recorded size, including the
header.
+ // A gap can truncate a projected page even when another file supplies the
filter.
+ pages.iter().enumerate().all(|(index, page)| {
+ let (next_row, next_byte) = pages.get(index + 1).map_or((row_count,
chunk_end), |next| {
+ (next.first_row_index, next.offset)
+ });
+ page.first_row_index < next_row
+ && page.compressed_page_size > 0
+ && page
+ .offset
+ .checked_add(i64::from(page.compressed_page_size))
+ .is_some_and(|end| end == next_byte)
+ && column_index.is_none_or(|stats| {
Review Comment:
Fixed in
[789b8d6c](https://github.com/apache/paimon-rust/commit/789b8d6c3523a5e2baf594b6c7e4fd11d9a76cc9).
Merged statistics pruning requires independently checkable page row counts
before using OffsetIndex boundaries. If those counts are unavailable, the
reader drops the optional OffsetIndex and decodes whole row-group column chunks
before selecting rows. Projected providers are checked even when pruning leaves
the caller's sparse selection unchanged.
The
[regression](https://github.com/apache/paimon-rust/blob/789b8d6c3523a5e2baf594b6c7e4fd11d9a76cc9/crates/paimon/src/table/data_evolution_reader/candidate_ranges_tests.rs#L793)
reproduces the 37-row `id` pages, one-row `rating` pages and incorrect
second-page start of 36. It covers both a missing ColumnIndex and one without
row-count histograms. The result remains `(id=36, _ROW_ID=100036)`, including
when the caller selects only row 100036.
##########
crates/paimon/src/arrow/format/parquet.rs:
##########
@@ -88,9 +94,90 @@ impl ParquetFormatReader {
page_index_enabled:
core_options.parquet_filter_column_index_enabled()?,
// Format-table files may be replaced in place.
metadata_cache_enabled: !core_options.is_format_table(),
+ data_evolution_enabled: core_options.data_evolution_enabled(),
+ metadata: None,
})
}
+ pub(crate) fn with_metadata(mut self, metadata: ArrowReaderMetadata) ->
Self {
+ self.metadata = Some(metadata);
+ self
+ }
+
+ fn reader_options(&self, has_predicate: bool, has_selection: bool) ->
ArrowReaderOptions {
+ let mut options = ArrowReaderOptions::new();
+ if self.page_index_enabled {
+ if has_predicate || (self.data_evolution_enabled && has_selection)
{
Review Comment:
Fixed in
[789b8d6c](https://github.com/apache/paimon-rust/commit/789b8d6c3523a5e2baf594b6c7e4fd11d9a76cc9).
Predicate-free sparse reads load only OffsetIndex with merged-read pruning
either enabled or disabled. If optional ColumnIndex loading fails, the reader
retries offsets independently before a footer-only fallback, reusing
already-fetched metadata bytes. Required ColumnIndex errors still propagate.
The [counting-I/O
regression](https://github.com/apache/paimon-rust/blob/789b8d6c3523a5e2baf594b6c7e4fd11d9a76cc9/crates/paimon/src/arrow/format/parquet.rs#L6975)
uses 30,000 one-row pages so indexes extend beyond footer prefetch. Valid and
corrupt optional ColumnIndexes produce the same row and metadata/data byte
counts as the ordinary OffsetIndex-only path. The [retry
regression](https://github.com/apache/paimon-rust/blob/789b8d6c3523a5e2baf594b6c7e4fd11d9a76cc9/crates/paimon/src/arrow/format/parquet.rs#L7039)
confirms that optional ColumnIndex failure preserves valid offsets with one
metadata read.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]