SteNicholas commented on code in PR #1469:
URL: https://github.com/apache/paimon-rust/pull/1469#discussion_r4656859051


##########
crates/paimon/src/arrow/format/parquet.rs:
##########
@@ -2807,51 +2834,96 @@ fn build_predicate_page_selection(
         });
     }
 
-    Ok(combined)
+    combined
 }
 
-/// Collect the file-field indices referenced by leaf predicates (recursing
-/// through compound nodes). Indices are already file-level (see caller).
-fn collect_leaf_field_indices(predicate: &Predicate, out: &mut Vec<usize>) {
-    match predicate {
-        Predicate::NestedLeaf { index, .. } => out.push(*index),
-        Predicate::Leaf { index, .. } => out.push(*index),
-        Predicate::And(children) | Predicate::Or(children) => {
-            for child in children {
-                collect_leaf_field_indices(child, out);
-            }
-        }
-        Predicate::Not(child) => collect_leaf_field_indices(child, out),
-        Predicate::AlwaysTrue | Predicate::AlwaysFalse => {}
-    }
+fn offset_index_valid(metadata: &ParquetMetaData, file_size: u64) -> bool {
+    let Some(groups) = metadata.offset_index() else {
+        return true;
+    };
+    groups.len() == metadata.num_row_groups()
+        && groups.iter().enumerate().all(|(group_index, offsets)| {
+            let group = metadata.row_group(group_index);
+            let stats = metadata
+                .column_index()
+                .and_then(|index| index.get(group_index));
+            offsets.len() == group.num_columns()
+                && offsets.iter().enumerate().all(|(column_index, offset)| {
+                    page_locations_valid(
+                        offset.page_locations(),
+                        group.num_rows(),
+                        group.column(column_index),
+                        file_size,
+                        stats.and_then(|columns| columns.get(column_index)),
+                    )
+                })
+        })
 }
 
-/// Validate that an OffsetIndex's page row boundaries are well-formed for a 
row
-/// group of `rg_rows` rows: the first page starts at row 0, `first_row_index`
-/// is strictly increasing, and every value stays within `[0, rg_rows]`.
-///
-/// Malformed metadata (negative, non-monotonic, or out-of-range boundaries)
-/// would otherwise produce invalid row ranges — huge values from `i64 as 
usize`
-/// underflow, or ranges past the row group — that panic
-/// `RowSelection::from_consecutive_ranges`. Callers fail open when this 
returns
-/// `false`.
-fn page_boundaries_valid(
+fn page_locations_valid(
     pages: &[parquet::file::page_index::offset_index::PageLocation],
-    rg_rows: usize,
+    row_count: i64,
+    column: &ColumnChunkMetaData,
+    file_size: u64,
+    column_index: Option<&ColumnIndexMetaData>,
 ) -> bool {
-    let rg_rows = rg_rows as i64;
-    let mut prev: i64 = -1;
-    for (page_idx, page) in pages.iter().enumerate() {
-        let first_row = page.first_row_index;
-        if page_idx == 0 && first_row != 0 {
-            return false;
-        }
-        if first_row <= prev || first_row > rg_rows {
-            return false;
-        }
-        prev = first_row;
+    let column_index = column_index.filter(|index| !matches!(index, 
ColumnIndexMetaData::NONE));
+    if column_index.is_some_and(|index| index.num_pages() != pages.len() as 
u64) {
+        return false;
+    }
+    let Some(first) = pages.first() else {
+        return row_count == 0;
+    };
+    let data_start = column.data_page_offset();
+    let chunk_start = column.dictionary_page_offset().unwrap_or(data_start);
+    let Some(chunk_end) = chunk_start.checked_add(column.compressed_size()) 
else {
+        return false;
+    };
+    if !(0..=data_start).contains(&chunk_start)
+        || !u64::try_from(chunk_end).is_ok_and(|end| end <= file_size)
+        || first.first_row_index != 0
+        || first.offset != data_start
+    {
+        return false;
+    }
+    // Arrow reads each indexed page by its recorded size, including the 
header.
+    // A gap can truncate a projected page even when another file supplies the 
filter.
+    pages.iter().enumerate().all(|(index, page)| {
+        let (next_row, next_byte) = pages.get(index + 1).map_or((row_count, 
chunk_end), |next| {
+            (next.first_row_index, next.offset)
+        });
+        page.first_row_index < next_row
+            && page.compressed_page_size > 0
+            && page
+                .offset
+                .checked_add(i64::from(page.compressed_page_size))
+                .is_some_and(|end| end == next_byte)
+            && column_index.is_none_or(|stats| {

Review Comment:
   Fixed in 
[b6d10803](https://github.com/apache/paimon-rust/commit/b6d10803a153c667a92434148659e054607f95a9).
 Merged pruning now requires independently checkable page row counts. When they 
are unavailable, it removes the provider's optional OffsetIndex and decodes 
whole row-group column chunks before selecting rows. Projected providers are 
also checked when the candidate range equals the caller's existing sparse 
selection.
   
   The regression uses 37-row `id` pages and one-row `rating` pages, then 
changes the second `id` page's first row from 37 to 36. It covers both an 
OffsetIndex-only file and a ColumnIndex without row-count histograms. The 
result stays `(id=36, _ROW_ID=100036)`, including when row 100036 is already 
selected. [Regression 
test](https://github.com/apache/paimon-rust/blob/b6d10803a153c667a92434148659e054607f95a9/crates/paimon/src/table/data_evolution_reader/candidate_ranges_tests.rs#L793)
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to