This is an automated email from the ASF dual-hosted git repository.
etseidl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 0219b5b754 Parquet: Make page index decoders public (#10899)
0219b5b754 is described below
commit 0219b5b754e68ad6c7b7c65a084320cdf7126721
Author: Ed Seidl <[email protected]>
AuthorDate: Sat Aug 29 07:32:07 2026 -0700
Parquet: Make page index decoders public (#10899)
# Which issue does this PR close?
- Part of #7582.
# Rationale for this change
To enable selective population of the page index, there must be a
mechanism to go from Thrift encoded bytes to crate index structures
(`ColumnIndexMetaData`, `OffsetIndexMetaData`). All of the functions in
this crate to do this transformation are private. This PR makes the
minimum set of functions necessary public to enable users to parse these
structures themselves.
# What changes are included in this PR?
Changes `decode_offset_index` and `decode_column_index` from the
`file::page_index` modules public, as well as `offset_index_range` and
`column_index_range` from `ColumnChunkMetaData`. Claude Code graciously
provided the public API documentation and examples.
This PR also makes some drive-by fixes to the `ColumnChunkMetaData`
documentation.
# Are these changes tested?
Covered by existing tests
# Are there any user-facing changes?
Adds 4 functions to the public API, but no breaking changes.
---
parquet/src/file/metadata/mod.rs | 20 +++--
parquet/src/file/page_index/index_reader.rs | 122 +++++++++++++++++++++++++++-
2 files changed, 134 insertions(+), 8 deletions(-)
diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs
index 4eeed39088..f81ae4fb35 100644
--- a/parquet/src/file/metadata/mod.rs
+++ b/parquet/src/file/metadata/mod.rs
@@ -1342,13 +1342,17 @@ impl ColumnChunkMetaData {
self.column_index_offset
}
- /// Returns the offset for the column index length.
+ /// Returns the length for the column index.
pub fn column_index_length(&self) -> Option<i32> {
self.column_index_length
}
- /// Returns the range for the offset index if any
- pub(crate) fn column_index_range(&self) -> Option<Range<u64>> {
+ /// Returns the range for the column index, if any.
+ ///
+ /// The range is `[column_index_offset, column_index_offset +
column_index_length)`,
+ /// where the offset is relative to the start of the Parquet file in which
the index
+ /// resides.
+ pub fn column_index_range(&self) -> Option<Range<u64>> {
let offset = u64::try_from(self.column_index_offset?).ok()?;
let length = u64::try_from(self.column_index_length?).ok()?;
Some(offset..(offset + length))
@@ -1359,13 +1363,17 @@ impl ColumnChunkMetaData {
self.offset_index_offset
}
- /// Returns the offset for the offset index length.
+ /// Returns the length for the offset index.
pub fn offset_index_length(&self) -> Option<i32> {
self.offset_index_length
}
- /// Returns the range for the offset index if any
- pub(crate) fn offset_index_range(&self) -> Option<Range<u64>> {
+ /// Returns the range for the offset index, if any.
+ ///
+ /// The range is `[offset_index_offset, offset_index_offset +
offset_index_length)`,
+ /// where the offset is relative to the start of the Parquet file in which
the index
+ /// resides.
+ pub fn offset_index_range(&self) -> Option<Range<u64>> {
let offset = u64::try_from(self.offset_index_offset?).ok()?;
let length = u64::try_from(self.offset_index_length?).ok()?;
Some(offset..(offset + length))
diff --git a/parquet/src/file/page_index/index_reader.rs
b/parquet/src/file/page_index/index_reader.rs
index 1c8c607554..963ee0ea97 100644
--- a/parquet/src/file/page_index/index_reader.rs
+++ b/parquet/src/file/page_index/index_reader.rs
@@ -42,7 +42,62 @@ pub(crate) fn acc_range(a: Option<Range<u64>>, b:
Option<Range<u64>>) -> Option<
}
}
-pub(crate) fn decode_offset_index(data: &[u8]) -> Result<OffsetIndexMetaData,
ParquetError> {
+/// Decode a Thrift [`OffsetIndex`] from the provided bytes.
+///
+/// The passed in bytes contain a serialized Thrift `OffsetIndex` struct as
+/// read from a Parquet file.
+///
+/// Returns an [`OffsetIndexMetaData`] containing page location information.
+///
+/// # Example
+///
+/// ```
+/// # use parquet::file::reader::{FileReader, SerializedFileReader};
+/// # use parquet::file::page_index::index_reader::decode_offset_index;
+/// # use std::fs::File;
+/// # use std::io::{Read, Seek};
+/// # use parquet::errors::Result;
+/// #
+/// # fn read_offset_index() -> Result<()> {
+/// // Open the Parquet file
+/// let mut file = File::open("data.parquet")?;
+/// let reader = SerializedFileReader::new(file.try_clone()?)?;
+/// let metadata = reader.metadata();
+///
+/// // Select a row group and column to read
+/// let row_group_idx = 0;
+/// let column_idx = 0;
+///
+/// // Get the column chunk metadata
+/// let row_group = metadata.row_group(row_group_idx);
+/// let column_chunk = row_group.column(column_idx);
+///
+/// // Get the offset index byte range from the column metadata
+/// if let Some(range) = column_chunk.offset_index_range() {
+/// // Read the offset index bytes from the file
+/// let mut buffer = vec![0u8; (range.end - range.start) as usize];
+/// file.seek(std::io::SeekFrom::Start(range.start))?;
+/// file.read_exact(&mut buffer)?;
+///
+/// // Decode the offset index
+/// let offset_index = decode_offset_index(&buffer)?;
+///
+/// // Access page location information
+/// for (i, page_location) in
offset_index.page_locations().iter().enumerate() {
+/// println!("Page {}: offset={}, size={}, first_row={}",
+/// i,
+/// page_location.offset,
+/// page_location.compressed_page_size,
+/// page_location.first_row_index
+/// );
+/// }
+/// }
+/// # Ok(())
+/// # }
+/// ```
+///
+/// [`OffsetIndex`]:
https://github.com/apache/parquet-format/blob/e94a5d090b324a0c0ee1adbb8ea6b099852dc3cc/src/main/thrift/parquet.thrift#L1253-L1273
+pub fn decode_offset_index(data: &[u8]) -> Result<OffsetIndexMetaData,
ParquetError> {
let mut prot = ThriftSliceInputProtocol::new(data);
// Try to read fast-path first. If that fails, fall back to slower but
more robust
@@ -70,7 +125,70 @@ pub(super) struct ThriftColumnIndex<'a> {
}
);
-pub(crate) fn decode_column_index(
+/// Decode a Thrift [`ColumnIndex`] from the provided bytes.
+///
+/// The passed in bytes contain a serialized Thrift `OffsetIndex` struct as
+/// read from a Parquet file. The `column_type` can be obtained via
+/// [`ColumnChunkMetaData::column_type`].
+///
+/// Returns a [`ColumnIndexMetaData`] containing per-page statistics.
+///
+/// # Example
+///
+/// ```
+/// # use parquet::file::reader::{FileReader, SerializedFileReader};
+/// # use parquet::file::page_index::index_reader::decode_column_index;
+/// # use std::fs::File;
+/// # use std::io::{Read, Seek};
+/// # use parquet::errors::Result;
+/// #
+/// # fn read_column_index() -> Result<()> {
+/// // Open the Parquet file
+/// let mut file = File::open("data.parquet")?;
+/// let reader = SerializedFileReader::new(file.try_clone()?)?;
+/// let metadata = reader.metadata();
+///
+/// // Select a row group and column to read
+/// let row_group_idx = 0;
+/// let column_idx = 0;
+///
+/// // Get the column chunk metadata
+/// let row_group = metadata.row_group(row_group_idx);
+/// let column_chunk = row_group.column(column_idx);
+///
+/// // Get the column index byte range from the column metadata
+/// if let Some(range) = column_chunk.column_index_range() {
+/// // Get the column type for proper deserialization
+/// let column_type = column_chunk.column_type();
+///
+/// // Read the column index bytes from the file
+/// let mut buffer = vec![0u8; (range.end - range.start) as usize];
+/// file.seek(std::io::SeekFrom::Start(range.start))?;
+/// file.read_exact(&mut buffer)?;
+///
+/// // Decode the column index
+/// let column_index = decode_column_index(&buffer, column_type)?;
+///
+/// // Access per-page statistics (example for INT32 column)
+/// use parquet::file::page_index::column_index::ColumnIndexMetaData;
+/// match column_index {
+/// ColumnIndexMetaData::INT32(index) => {
+/// for (i, (min, max)) in index.min_values().iter()
+/// .zip(index.max_values().iter())
+/// .enumerate() {
+/// println!("Page {}: min={}, max={}", i, min, max);
+/// }
+/// }
+/// _ => println!("Column is not INT32 type"),
+/// }
+/// }
+/// # Ok(())
+/// # }
+/// ```
+///
+/// [`ColumnChunkMetaData::column_type`]:
crate::file::metadata::ColumnChunkMetaData::column_type
+/// [`ColumnIndex`]:
https://github.com/apache/parquet-format/blob/e94a5d090b324a0c0ee1adbb8ea6b099852dc3cc/src/main/thrift/parquet.thrift#L1275-1373
+pub fn decode_column_index(
data: &[u8],
column_type: Type,
) -> Result<ColumnIndexMetaData, ParquetError> {
