etseidl commented on code in PR #10719: URL: https://github.com/apache/arrow-rs/pull/10719#discussion_r3799526498
########## parquet/src/file/metadata/mod.rs: ########## @@ -134,36 +134,123 @@ use std::sync::Arc; pub use writer::ParquetMetaDataWriter; pub(crate) use writer::ThriftMetadataWriter; -/// Page level statistics for each column chunk of each row group. +/// Struct to encapsulate the Parquet [Page Index] /// -/// This structure is an in-memory representation of multiple [`ColumnIndex`] -/// structures in a parquet file footer, as described in the Parquet [PageIndex -/// documentation]. Each [`ColumnIndex`] holds statistics about all the pages in a -/// particular column chunk. +/// The Parquet page index comprises two structures, the [`ColumnIndex`] +/// and [`OffsetIndex`]. The column index contains per-page statistics for a given +/// row group and column chunk, while the offset index gives page location and size +/// information for the same. The column index can be used to filter pages based on +/// a predicate; pages that pass can then be located via the offset index. /// -/// `column_index[row_group_number][column_number]` holds the -/// [`ColumnIndex`] corresponding to column `column_number` of row group -/// `row_group_number`. -/// -/// For example `column_index[2][3]` holds the [`ColumnIndex`] for the fourth -/// column in the third row group of the parquet file. -/// -/// [PageIndex documentation]: https://github.com/apache/parquet-format/blob/master/PageIndex.md +/// [Page Index]: https://github.com/apache/parquet-format/blob/master/PageIndex.md /// [`ColumnIndex`]: crate::file::page_index::column_index::ColumnIndexMetaData -pub type ParquetColumnIndex = Vec<Vec<ColumnIndexMetaData>>; +/// [`OffsetIndex`]: crate::file::page_index::offset_index::OffsetIndexMetaData +#[derive(Debug, Clone, PartialEq)] +pub struct PageIndex { + column_indexes: Option<Vec<Vec<Option<ColumnIndexMetaData>>>>, + offset_indexes: Option<Vec<Vec<Option<OffsetIndexMetaData>>>>, +} -/// [`OffsetIndexMetaData`] for each data page of each row group of each column -/// -/// This structure is the parsed representation of the [`OffsetIndex`] from the -/// Parquet file footer, as described in the Parquet [PageIndex documentation]. -/// -/// `offset_index[row_group_number][column_number]` holds -/// the [`OffsetIndexMetaData`] corresponding to column -/// `column_number`of row group `row_group_number`. -/// -/// [PageIndex documentation]: https://github.com/apache/parquet-format/blob/master/PageIndex.md -/// [`OffsetIndex`]: https://github.com/apache/parquet-format/blob/master/PageIndex.md -pub type ParquetOffsetIndex = Vec<Vec<OffsetIndexMetaData>>; +impl PageIndex { Review Comment: TODO: better documentation -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
