etseidl commented on code in PR #10842: URL: https://github.com/apache/arrow-rs/pull/10842#discussion_r3918365786
########## parquet/src/file/metadata/page_index.rs: ########## @@ -0,0 +1,672 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Page Index structures for efficient page-level skipping + +use crate::file::metadata::memory::HeapSize; +use crate::file::page_index::{ + column_index::ColumnIndexMetaData, + offset_index::{OffsetIndexMetaData, PageLocation}, +}; +use std::sync::Arc; + +/// Trait for accessing Parquet [Page Index] data for efficient page-level skipping +/// +/// The [Page Index] enables query engines to skip irrelevant data pages during scans, +/// significantly improving I/O efficiency. It provides access to two complementary +/// structures: +/// +/// * **[`ColumnIndex`]**: Per-page min/max value boundaries that enable predicate-based +/// page filtering. Allows determining which pages might contain rows matching a query +/// predicate without reading the actual data pages. +/// +/// * **[`OffsetIndex`]**: Physical locations and sizes of data pages, plus the first row +/// index of each page. Used to locate and read only the pages identified as relevant +/// by the ColumnIndex. +/// +/// Together, these indexes enable: +/// - Single-row lookups reading only one data page per column (on sorted columns) +/// - Range scans reading only pages containing values in the query range +/// - Efficient cross-column filtering by skipping corresponding row ranges +/// +/// # Structure +/// +/// Within a Parquet file, both indexes are organized as a two-level structure, with +/// indexes arranged first by row group, and then column. The [`ColumnChunkMetaData`] +/// contains pointers to the indexes for a given column chunk, so they may be +/// populated piecemeal. This trait allows access by row group index and column number +/// ([Self::column_index], [Self::offset_index]). Access by row group is provided by +/// [`RowGroupPageIndex`]. +/// +/// Each entry is `Option<T>` because: +/// - The entire page index might be absent (old files, disabled during write) +/// - Individual columns might lack indexes (unsupported types, statistics disabled) +/// +/// # Example: Checking if Page Index is Available +/// +/// ``` +/// use parquet::file::metadata::ParquetMetaData; +/// # use parquet::errors::Result; +/// +/// fn check_page_index_availability(metadata: &ParquetMetaData) -> Result<()> { +/// if let Some(page_index) = metadata.page_index() { +/// println!("Page index present:"); +/// println!(" Has offset indexes: {}", page_index.has_offset_indexes()); +/// println!(" Has column indexes: {}", page_index.has_column_indexes()); +/// +/// // Check availability for first row group, first column +/// if let Some(col_idx) = page_index.column_index(0, 0) { +/// println!(" Column index found for row group 0, column 0"); +/// println!(" Number of pages: {}", col_idx.num_pages()); +/// } +/// +/// if let Some(offset_idx) = page_index.offset_index(0, 0) { +/// println!(" Offset index found for row group 0, column 0"); +/// println!(" Number of pages: {}", offset_idx.page_locations().len()); +/// } +/// } else { +/// println!("No page index available"); +/// } +/// Ok(()) +/// } +/// ``` +/// +/// # Example: Using Page Index for Predicate Pushdown +/// +/// ``` +/// use parquet::file::metadata::ParquetMetaData; +/// use parquet::file::page_index::column_index::ColumnIndexMetaData; +/// # use parquet::errors::Result; +/// +/// /// Identifies which pages in a column might contain values >= min_value +/// fn find_relevant_pages( +/// metadata: &ParquetMetaData, +/// row_group_idx: usize, +/// column_idx: usize, +/// min_value: i32, +/// ) -> Vec<usize> { +/// let mut relevant_pages = Vec::new(); +/// +/// let Some(page_index) = metadata.page_index() else { +/// // No page index - must read all pages +/// return relevant_pages; +/// }; +/// +/// let Some(column_index) = page_index.column_index(row_group_idx, column_idx) else { +/// // No column index - must read all pages +/// return relevant_pages; +/// }; +/// +/// // Check each page's statistics +/// match column_index { +/// ColumnIndexMetaData::INT32(index) => { +/// for (page_num, max_value) in index.max_values_iter().enumerate() { +/// // Page might contain matching rows if its max >= our min +/// if let Some(max) = max_value { +/// if *max >= min_value { +/// relevant_pages.push(page_num); +/// } +/// } +/// } +/// } +/// _ => { +/// // Wrong column type - read all pages +/// } +/// } +/// +/// relevant_pages +/// } +/// ``` +/// +/// [Page Index]: https://parquet.apache.org/docs/file-format/pageindex/ +/// [`ColumnIndex`]: crate::file::page_index::column_index::ColumnIndexMetaData +/// [`OffsetIndex`]: crate::file::page_index::offset_index::OffsetIndexMetaData +/// [`ColumnChunkMetaData`]: crate::file::metadata::ColumnChunkMetaData +pub trait PageIndexProvider: Send + Sync + std::fmt::Debug { + /// Returns `true` if offset index structures are present Review Comment: attempted fix in https://github.com/apache/arrow-rs/pull/10842/commits/39c5828abbf3e465b62ab5d2cb2838a8900d20ac -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
