SteNicholas commented on code in PR #1469:
URL: https://github.com/apache/paimon-rust/pull/1469#discussion_r4653626635


##########
crates/paimon/src/arrow/format/parquet.rs:
##########
@@ -2807,51 +2827,41 @@ fn build_predicate_page_selection(
         });
     }
 
-    Ok(combined)
-}
-
-/// Collect the file-field indices referenced by leaf predicates (recursing
-/// through compound nodes). Indices are already file-level (see caller).
-fn collect_leaf_field_indices(predicate: &Predicate, out: &mut Vec<usize>) {
-    match predicate {
-        Predicate::NestedLeaf { index, .. } => out.push(*index),
-        Predicate::Leaf { index, .. } => out.push(*index),
-        Predicate::And(children) | Predicate::Or(children) => {
-            for child in children {
-                collect_leaf_field_indices(child, out);
-            }
-        }
-        Predicate::Not(child) => collect_leaf_field_indices(child, out),
-        Predicate::AlwaysTrue | Predicate::AlwaysFalse => {}
-    }
+    combined
 }
 
-/// Validate that an OffsetIndex's page row boundaries are well-formed for a 
row
-/// group of `rg_rows` rows: the first page starts at row 0, `first_row_index`
-/// is strictly increasing, and every value stays within `[0, rg_rows]`.
-///
-/// Malformed metadata (negative, non-monotonic, or out-of-range boundaries)
-/// would otherwise produce invalid row ranges — huge values from `i64 as 
usize`
-/// underflow, or ranges past the row group — that panic
-/// `RowSelection::from_consecutive_ranges`. Callers fail open when this 
returns
-/// `false`.
-fn page_boundaries_valid(
+fn page_locations_valid(
     pages: &[parquet::file::page_index::offset_index::PageLocation],
-    rg_rows: usize,
+    row_count: i64,
+    column: &ColumnChunkMetaData,
+    file_size: u64,
 ) -> bool {
-    let rg_rows = rg_rows as i64;
-    let mut prev: i64 = -1;
-    for (page_idx, page) in pages.iter().enumerate() {
-        let first_row = page.first_row_index;
-        if page_idx == 0 && first_row != 0 {
-            return false;
-        }
-        if first_row <= prev || first_row > rg_rows {
-            return false;
-        }
-        prev = first_row;
+    let Some(first) = pages.first() else {
+        return row_count == 0;
+    };
+    let data_start = column.data_page_offset();
+    let chunk_start = column.dictionary_page_offset().unwrap_or(data_start);
+    let Some(chunk_end) = chunk_start.checked_add(column.compressed_size()) 
else {
+        return false;
+    };
+    if !(0..=data_start).contains(&chunk_start)
+        || !u64::try_from(chunk_end).is_ok_and(|end| end <= file_size)
+        || first.first_row_index != 0
+        || first.offset != data_start
+    {
+        return false;
     }
-    true
+    pages.iter().enumerate().all(|(index, page)| {
+        let (next_row, next_byte) = pages.get(index + 1).map_or((row_count, 
chunk_end), |next| {
+            (next.first_row_index, next.offset)
+        });
+        page.first_row_index < next_row
+            && page.compressed_page_size > 0
+            && page
+                .offset
+                .checked_add(i64::from(page.compressed_page_size))
+                .is_some_and(|end| end <= next_byte)

Review Comment:
   Fixed in 
[789b8d6c](https://github.com/apache/paimon-rust/commit/789b8d6c3523a5e2baf594b6c7e4fd11d9a76cc9).
 Each indexed data page must end where the next begins, and the last must end 
at the column-chunk boundary. Gaps or overlaps discard the optional OffsetIndex 
and fall back to whole row-group column chunks.
   
   The 
[regression](https://github.com/apache/paimon-rust/blob/789b8d6c3523a5e2baf594b6c7e4fd11d9a76cc9/crates/paimon/src/table/data_evolution_reader/candidate_ranges_tests.rs#L869)
 shortens the last page's recorded size by one byte while leaving the data 
intact, for both projected and filtered files. `rating = 4095` returns the 
expected row with pushdown enabled and disabled, including when the caller 
selects only the last two rows. The same test covers oversized entries, 
first-page truncation, gaps and overlaps.
   



##########
crates/paimon/src/arrow/format/parquet.rs:
##########
@@ -2807,51 +2834,96 @@ fn build_predicate_page_selection(
         });
     }
 
-    Ok(combined)
+    combined
 }
 
-/// Collect the file-field indices referenced by leaf predicates (recursing
-/// through compound nodes). Indices are already file-level (see caller).
-fn collect_leaf_field_indices(predicate: &Predicate, out: &mut Vec<usize>) {
-    match predicate {
-        Predicate::NestedLeaf { index, .. } => out.push(*index),
-        Predicate::Leaf { index, .. } => out.push(*index),
-        Predicate::And(children) | Predicate::Or(children) => {
-            for child in children {
-                collect_leaf_field_indices(child, out);
-            }
-        }
-        Predicate::Not(child) => collect_leaf_field_indices(child, out),
-        Predicate::AlwaysTrue | Predicate::AlwaysFalse => {}
-    }
+fn offset_index_valid(metadata: &ParquetMetaData, file_size: u64) -> bool {
+    let Some(groups) = metadata.offset_index() else {
+        return true;
+    };
+    groups.len() == metadata.num_row_groups()
+        && groups.iter().enumerate().all(|(group_index, offsets)| {
+            let group = metadata.row_group(group_index);
+            let stats = metadata
+                .column_index()
+                .and_then(|index| index.get(group_index));
+            offsets.len() == group.num_columns()
+                && offsets.iter().enumerate().all(|(column_index, offset)| {
+                    page_locations_valid(
+                        offset.page_locations(),
+                        group.num_rows(),
+                        group.column(column_index),
+                        file_size,
+                        stats.and_then(|columns| columns.get(column_index)),
+                    )
+                })
+        })
 }
 
-/// Validate that an OffsetIndex's page row boundaries are well-formed for a 
row
-/// group of `rg_rows` rows: the first page starts at row 0, `first_row_index`
-/// is strictly increasing, and every value stays within `[0, rg_rows]`.
-///
-/// Malformed metadata (negative, non-monotonic, or out-of-range boundaries)
-/// would otherwise produce invalid row ranges — huge values from `i64 as 
usize`
-/// underflow, or ranges past the row group — that panic
-/// `RowSelection::from_consecutive_ranges`. Callers fail open when this 
returns
-/// `false`.
-fn page_boundaries_valid(
+fn page_locations_valid(
     pages: &[parquet::file::page_index::offset_index::PageLocation],
-    rg_rows: usize,
+    row_count: i64,
+    column: &ColumnChunkMetaData,
+    file_size: u64,
+    column_index: Option<&ColumnIndexMetaData>,
 ) -> bool {
-    let rg_rows = rg_rows as i64;
-    let mut prev: i64 = -1;
-    for (page_idx, page) in pages.iter().enumerate() {
-        let first_row = page.first_row_index;
-        if page_idx == 0 && first_row != 0 {
-            return false;
-        }
-        if first_row <= prev || first_row > rg_rows {
-            return false;
-        }
-        prev = first_row;
+    let column_index = column_index.filter(|index| !matches!(index, 
ColumnIndexMetaData::NONE));
+    if column_index.is_some_and(|index| index.num_pages() != pages.len() as 
u64) {
+        return false;
+    }
+    let Some(first) = pages.first() else {
+        return row_count == 0;
+    };
+    let data_start = column.data_page_offset();
+    let chunk_start = column.dictionary_page_offset().unwrap_or(data_start);
+    let Some(chunk_end) = chunk_start.checked_add(column.compressed_size()) 
else {
+        return false;
+    };
+    if !(0..=data_start).contains(&chunk_start)
+        || !u64::try_from(chunk_end).is_ok_and(|end| end <= file_size)
+        || first.first_row_index != 0
+        || first.offset != data_start
+    {
+        return false;
+    }
+    // Arrow reads each indexed page by its recorded size, including the 
header.
+    // A gap can truncate a projected page even when another file supplies the 
filter.
+    pages.iter().enumerate().all(|(index, page)| {
+        let (next_row, next_byte) = pages.get(index + 1).map_or((row_count, 
chunk_end), |next| {
+            (next.first_row_index, next.offset)
+        });
+        page.first_row_index < next_row
+            && page.compressed_page_size > 0
+            && page
+                .offset
+                .checked_add(i64::from(page.compressed_page_size))
+                .is_some_and(|end| end == next_byte)
+            && column_index.is_none_or(|stats| {

Review Comment:
   Fixed in 
[789b8d6c](https://github.com/apache/paimon-rust/commit/789b8d6c3523a5e2baf594b6c7e4fd11d9a76cc9).
 Merged statistics pruning requires independently checkable page row counts 
before using OffsetIndex boundaries. If those counts are unavailable, the 
reader drops the optional OffsetIndex and decodes whole row-group column chunks 
before selecting rows. Projected providers are checked even when pruning leaves 
the caller's sparse selection unchanged.
   
   The 
[regression](https://github.com/apache/paimon-rust/blob/789b8d6c3523a5e2baf594b6c7e4fd11d9a76cc9/crates/paimon/src/table/data_evolution_reader/candidate_ranges_tests.rs#L793)
 reproduces the 37-row `id` pages, one-row `rating` pages and incorrect 
second-page start of 36. It covers both a missing ColumnIndex and one without 
row-count histograms. The result remains `(id=36, _ROW_ID=100036)`, including 
when the caller selects only row 100036.
   



##########
crates/paimon/src/arrow/format/parquet.rs:
##########
@@ -88,9 +94,90 @@ impl ParquetFormatReader {
             page_index_enabled: 
core_options.parquet_filter_column_index_enabled()?,
             // Format-table files may be replaced in place.
             metadata_cache_enabled: !core_options.is_format_table(),
+            data_evolution_enabled: core_options.data_evolution_enabled(),
+            metadata: None,
         })
     }
 
+    pub(crate) fn with_metadata(mut self, metadata: ArrowReaderMetadata) -> 
Self {
+        self.metadata = Some(metadata);
+        self
+    }
+
+    fn reader_options(&self, has_predicate: bool, has_selection: bool) -> 
ArrowReaderOptions {
+        let mut options = ArrowReaderOptions::new();
+        if self.page_index_enabled {
+            if has_predicate || (self.data_evolution_enabled && has_selection) 
{

Review Comment:
   Fixed in 
[789b8d6c](https://github.com/apache/paimon-rust/commit/789b8d6c3523a5e2baf594b6c7e4fd11d9a76cc9).
 Predicate-free sparse reads load only OffsetIndex with merged-read pruning 
either enabled or disabled. If optional ColumnIndex loading fails, the reader 
retries offsets independently before a footer-only fallback, reusing 
already-fetched metadata bytes. Required ColumnIndex errors still propagate.
   
   The [counting-I/O 
regression](https://github.com/apache/paimon-rust/blob/789b8d6c3523a5e2baf594b6c7e4fd11d9a76cc9/crates/paimon/src/arrow/format/parquet.rs#L6975)
 uses 30,000 one-row pages so indexes extend beyond footer prefetch. Valid and 
corrupt optional ColumnIndexes produce the same row and metadata/data byte 
counts as the ordinary OffsetIndex-only path. The [retry 
regression](https://github.com/apache/paimon-rust/blob/789b8d6c3523a5e2baf594b6c7e4fd11d9a76cc9/crates/paimon/src/arrow/format/parquet.rs#L7039)
 confirms that optional ColumnIndex failure preserves valid offsets with one 
metadata read.
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to