This is an automated email from the ASF dual-hosted git repository.
etseidl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 34060c4120 Allow for serialization of Parquet Page Index via custom
`PageIndexProvider` (#11031)
34060c4120 is described below
commit 34060c412044f891998e7288fb6d7b4e3fc7867d
Author: Ed Seidl <[email protected]>
AuthorDate: Thu Sep 24 13:26:07 2026 -0700
Allow for serialization of Parquet Page Index via custom
`PageIndexProvider` (#11031)
# Which issue does this PR close?
- Closes #11030.
# Rationale for this change
See issue, but allow for serialization of the page indexes through the
`PageIndexProvider` API.
# What changes are included in this PR?
Changes `ThriftMetadataWriter` to operate through the
`PageIndexProvider` API rather than using individual column and offset
index vectors. Removes some now unnecessary `pub(crate)` functions.
# Are these changes tested?
Yes
# Are there any user-facing changes?
All page indexes will now serialize, but no changes to the public API
---
parquet/src/arrow/mod.rs | 116 ++++++++++++++++++++
parquet/src/file/metadata/page_index.rs | 16 ---
parquet/src/file/metadata/writer.rs | 182 ++++++++++++++------------------
parquet/src/file/writer.rs | 74 ++++++++++++-
4 files changed, 266 insertions(+), 122 deletions(-)
diff --git a/parquet/src/arrow/mod.rs b/parquet/src/arrow/mod.rs
index e2ea1b4ca0..e5cbafea5e 100644
--- a/parquet/src/arrow/mod.rs
+++ b/parquet/src/arrow/mod.rs
@@ -595,6 +595,112 @@ mod test {
assert_eq!(original_metadata, roundtrip_metadata);
}
+ #[test]
+ fn test_metadata_read_write_roundtrip_missing_page_index() {
+ let parquet_bytes = create_parquet_file();
+
+ // read the metadata from the file but skip the page indexes
+ let options =
ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false);
+ let original_metadata = ParquetMetaDataReader::new()
+ .with_metadata_options(Some(options))
+ .with_page_index_policy(PageIndexPolicy::Skip)
+ .parse_and_finish(&parquet_bytes)
+ .unwrap();
+
+ // metadata_to_bytes_no_page_idx should zero out the page index
locations. if they aren't
+ // then reading metadata_bytes will fail with EOF
+ let metadata_bytes = metadata_to_bytes_no_page_idx(&original_metadata);
+ let options =
ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false);
+ let roundtrip_metadata = ParquetMetaDataReader::new()
+ .with_metadata_options(Some(options))
+ .with_page_index_policy(PageIndexPolicy::Optional)
+ .parse_and_finish(&metadata_bytes)
+ .expect("page index locations should have been cleared");
+
+ assert!(roundtrip_metadata.page_index().is_none());
+ }
+
+ #[test]
+ fn test_metadata_read_write_roundtrip_offset_index_only() {
+ // `Chunk` statistics: the file has an offset index, but no column
index
+ let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+ let batch = RecordBatch::try_from_iter(vec![("id", array)]).unwrap();
+ let props = WriterProperties::builder()
+ .set_statistics_enabled(EnabledStatistics::Chunk)
+ .build();
+ let mut buf = vec![];
+ let mut writer = ArrowWriter::try_new(&mut buf, batch.schema(),
Some(props)).unwrap();
+ writer.write(&batch).unwrap();
+ writer.close().unwrap();
+
+ let read = |bytes: &Bytes| {
+ let options =
ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false);
+ ParquetMetaDataReader::new()
+ .with_metadata_options(Some(options))
+ .with_page_index_policy(PageIndexPolicy::Optional)
+ .parse_and_finish(bytes)
+ .unwrap()
+ };
+ let original = read(&Bytes::from(buf));
+ let page_index = original.page_index().unwrap();
+ assert!(!page_index.has_column_indexes() &&
page_index.has_offset_indexes());
+ let roundtrip = read(&metadata_to_bytes(&original));
+ assert_eq!(
+ normalize_locations(original),
+ normalize_locations(roundtrip)
+ );
+ }
+
+ #[test]
+ fn test_metadata_read_write_roundtrip_custom_page_index() {
+ use crate::file::metadata::page_index::PageIndexProvider;
+ use crate::file::page_index::{
+ column_index::ColumnIndexMetaData,
offset_index::OffsetIndexMetaData,
+ };
+
+ /// A custom provider that forwards to another provider
+ #[derive(Debug)]
+ struct Forward(Arc<dyn PageIndexProvider>);
+ impl PageIndexProvider for Forward {
+ fn has_offset_indexes(&self) -> bool {
+ self.0.has_offset_indexes()
+ }
+ fn has_column_indexes(&self) -> bool {
+ self.0.has_column_indexes()
+ }
+ fn column_index(&self, rg: usize, col: usize) ->
Option<&ColumnIndexMetaData> {
+ self.0.column_index(rg, col)
+ }
+ fn offset_index(&self, rg: usize, col: usize) ->
Option<&OffsetIndexMetaData> {
+ self.0.offset_index(rg, col)
+ }
+ fn as_any(&self) -> &dyn std::any::Any {
+ self
+ }
+ }
+
+ let read = |bytes: &Bytes| {
+ let options =
ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false);
+ ParquetMetaDataReader::new()
+ .with_metadata_options(Some(options))
+ .with_page_index_policy(PageIndexPolicy::Required)
+ .parse_and_finish(bytes)
+ .unwrap()
+ };
+ let original = read(&create_parquet_file());
+ let provider = Forward(original.page_index().unwrap().clone());
+ let custom = original
+ .clone()
+ .into_builder()
+ .set_page_index(Some(Arc::new(provider)))
+ .build();
+ let roundtrip = read(&metadata_to_bytes(&custom));
+ assert_eq!(
+ normalize_locations(original),
+ normalize_locations(roundtrip)
+ );
+ }
+
/// Sets the page index offset locations in the metadata to `None`
///
/// This is because the offsets are used to find the relative location of
the index
@@ -647,6 +753,16 @@ mod test {
Bytes::from(buf)
}
+ // like metadata_to_bytes, but do not preserve page index location info
+ fn metadata_to_bytes_no_page_idx(metadata: &ParquetMetaData) -> Bytes {
+ let mut buf = vec![];
+ ParquetMetaDataWriter::new(&mut buf, metadata)
+ .with_preserve_page_index_locations(false)
+ .finish()
+ .unwrap();
+ Bytes::from(buf)
+ }
+
#[test]
fn test_mask_from_column_names() {
let schema = parse_schema(
diff --git a/parquet/src/file/metadata/page_index.rs
b/parquet/src/file/metadata/page_index.rs
index 9ded57d0a2..78c3dc2792 100644
--- a/parquet/src/file/metadata/page_index.rs
+++ b/parquet/src/file/metadata/page_index.rs
@@ -460,22 +460,6 @@ impl PageIndex {
pub fn into_builder(self) -> PageIndexBuilder {
self.into()
}
-
- /// Returns a reference to the raw column indexes structure
- ///
- /// This method provides access to the underlying column index data for
serialization
- /// and other low-level operations.
- pub(crate) fn column_indexes_raw(&self) ->
Option<&Vec<Vec<Option<ColumnIndexMetaData>>>> {
- self.column_indexes.as_ref()
- }
-
- /// Returns a reference to the raw offset indexes structure
- ///
- /// This method provides access to the underlying offset index data for
serialization
- /// and other low-level operations.
- pub(crate) fn offset_indexes_raw(&self) ->
Option<&Vec<Vec<Option<OffsetIndexMetaData>>>> {
- self.offset_indexes.as_ref()
- }
}
impl PageIndexProvider for PageIndex {
diff --git a/parquet/src/file/metadata/writer.rs
b/parquet/src/file/metadata/writer.rs
index a1708ccf2c..3a00f855bc 100644
--- a/parquet/src/file/metadata/writer.rs
+++ b/parquet/src/file/metadata/writer.rs
@@ -15,8 +15,9 @@
// specific language governing permissions and limitations
// under the License.
+use crate::file::metadata::page_index::PageIndexProvider;
use crate::file::metadata::thrift::FileMeta;
-use crate::file::metadata::{ColumnChunkMetaData, PageIndex, RowGroupMetaData};
+use crate::file::metadata::{ColumnChunkMetaData, RowGroupMetaData};
use crate::schema::types::{SchemaDescPtr, SchemaDescriptor};
use crate::{
basic::ColumnOrder,
@@ -54,13 +55,13 @@ pub(crate) struct ThriftMetadataWriter<'a, W: Write> {
buf: &'a mut TrackedWrite<W>,
schema_descr: &'a SchemaDescPtr,
row_groups: Vec<RowGroupMetaData>,
- column_indexes: Option<Vec<Vec<Option<ColumnIndexMetaData>>>>,
- offset_indexes: Option<Vec<Vec<Option<OffsetIndexMetaData>>>>,
+ page_index: Option<Arc<dyn PageIndexProvider>>,
key_value_metadata: Option<Vec<KeyValue>>,
created_by: Option<String>,
object_writer: MetadataObjectWriter,
writer_version: i32,
write_path_in_schema: bool,
+ preserve_page_index_locations: bool,
}
impl<'a, W: Write> ThriftMetadataWriter<'a, W> {
@@ -69,16 +70,13 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> {
/// Note: also updates the `ColumnChunk::offset_index_offset` and
/// `ColumnChunk::offset_index_length` to reflect the position and length
/// of the serialized offset indexes.
- fn write_offset_indexes(
- &mut self,
- offset_indexes: &[Vec<Option<OffsetIndexMetaData>>],
- ) -> Result<()> {
+ fn write_offset_indexes(&mut self, page_index: &dyn PageIndexProvider) ->
Result<()> {
// iter row group
// iter each column
// write offset index to the file
for (row_group_idx, row_group) in
self.row_groups.iter_mut().enumerate() {
for (column_idx, column_metadata) in
row_group.columns.iter_mut().enumerate() {
- if let Some(offset_index) =
&offset_indexes[row_group_idx][column_idx] {
+ if let Some(offset_index) =
page_index.offset_index(row_group_idx, column_idx) {
let start_offset = self.buf.bytes_written();
self.object_writer.write_offset_index(
offset_index,
@@ -91,6 +89,9 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> {
// set offset and index for offset index
column_metadata.offset_index_offset = Some(start_offset as
i64);
column_metadata.offset_index_length = Some((end_offset -
start_offset) as i32);
+ } else if !self.preserve_page_index_locations {
+ column_metadata.offset_index_offset = None;
+ column_metadata.offset_index_length = None;
}
}
}
@@ -102,16 +103,13 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> {
/// Note: also updates the `ColumnChunk::column_index_offset` and
/// `ColumnChunk::column_index_length` to reflect the position and length
/// of the serialized column indexes.
- fn write_column_indexes(
- &mut self,
- column_indexes: &[Vec<Option<ColumnIndexMetaData>>],
- ) -> Result<()> {
+ fn write_column_indexes(&mut self, page_index: &dyn PageIndexProvider) ->
Result<()> {
// iter row group
// iter each column
// write column index to the file
for (row_group_idx, row_group) in
self.row_groups.iter_mut().enumerate() {
for (column_idx, column_metadata) in
row_group.columns.iter_mut().enumerate() {
- if let Some(column_index) =
&column_indexes[row_group_idx][column_idx] {
+ if let Some(column_index) =
page_index.column_index(row_group_idx, column_idx) {
let start_offset = self.buf.bytes_written();
// only update column_metadata if the write succeeds
if self.object_writer.write_column_index(
@@ -127,61 +125,39 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> {
column_metadata.column_index_length =
Some((end_offset - start_offset) as i32);
}
+ } else if !self.preserve_page_index_locations {
+ column_metadata.column_index_offset = None;
+ column_metadata.column_index_length = None;
}
}
}
Ok(())
}
- /// Serialize the column indexes and transform to
`Option<ParquetColumnIndex>`
- fn finalize_column_indexes(&mut self) ->
Result<Option<Vec<Vec<Option<ColumnIndexMetaData>>>>> {
- let column_indexes = std::mem::take(&mut self.column_indexes);
-
- // Write column indexes to file
- if let Some(column_indexes) = column_indexes.as_ref() {
- self.write_column_indexes(column_indexes)?;
- }
-
- // check to see if the index is `None` for every row group and column
chunk
- let all_none = column_indexes
- .as_ref()
- .is_some_and(|ci| ci.iter().all(|cii| cii.iter().all(|idx|
idx.is_none())));
-
- if all_none {
- Ok(None)
- } else {
- Ok(column_indexes)
- }
- }
-
- /// Serialize the offset indexes and transform to
`Option<ParquetOffsetIndex>`
- fn finalize_offset_indexes(&mut self) ->
Result<Option<Vec<Vec<Option<OffsetIndexMetaData>>>>> {
- let offset_indexes = std::mem::take(&mut self.offset_indexes);
-
- // Write offset indexes to file
- if let Some(offset_indexes) = offset_indexes.as_ref() {
- self.write_offset_indexes(offset_indexes)?;
- }
-
- // check to see if the index is `None` for every row group and column
chunk
- let all_none = offset_indexes
- .as_ref()
- .is_some_and(|oi| oi.iter().all(|oii| oii.iter().all(|idx|
idx.is_none())));
-
- if all_none {
- Ok(None)
- } else {
- Ok(offset_indexes)
+ fn clear_page_indexes(&mut self) {
+ for row_group in &mut self.row_groups {
+ for column_metadata in &mut row_group.columns {
+ column_metadata.column_index_offset = None;
+ column_metadata.column_index_length = None;
+ column_metadata.offset_index_offset = None;
+ column_metadata.offset_index_length = None;
+ }
}
}
/// Assembles and writes the final metadata to self.buf
- pub fn finish(mut self) -> Result<ParquetMetaData> {
+ pub(crate) fn finish(mut self) -> Result<ParquetMetaData> {
let num_rows = self.row_groups.iter().map(|x| x.num_rows).sum();
- // serialize page indexes and transform to the proper form for use in
ParquetMetaData
- let column_indexes = self.finalize_column_indexes()?;
- let offset_indexes = self.finalize_offset_indexes()?;
+ // serialize page indexes and update index locations
+ let page_index = self.page_index.take();
+ if let Some(pi) = page_index.as_deref() {
+ self.write_column_indexes(pi)?;
+ self.write_offset_indexes(pi)?;
+ } else if !self.preserve_page_index_locations {
+ // no page indexes at all, then clear index locations
+ self.clear_page_indexes();
+ }
// We only include ColumnOrder for leaf nodes.
let column_orders = self
@@ -249,65 +225,61 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> {
self.buf.write_all(&metadata_len.to_le_bytes())?;
self.buf.write_all(self.object_writer.get_file_magic())?;
+ // Note: we simply pass the original reference counted page index
provider to the
+ // builder. Either it came from ParquetMetaDataWriter where it was
cloned, and this
+ // reference will go away when ParquetMetaDataWriter::finish drops the
newly created
+ // metadata, or it was passed an owned version from
SerializedFileWriter which then
+ // takes it back via the returned metadata.
+ let builder =
ParquetMetaDataBuilder::new(file_metadata).set_page_index(page_index);
+
// If row group metadata was encrypted, we replace the encrypted row
groups with
// unencrypted metadata before it is returned to users. This allows
the metadata
// to be usable for retrieving the row group statistics for example,
without users
// needing to decrypt the metadata.
- let builder =
ParquetMetaDataBuilder::new(file_metadata).set_page_index(Some(Arc::new(
- PageIndex::new(column_indexes, offset_indexes),
- )));
-
Ok(match unencrypted_row_groups {
Some(rg) => builder.set_row_groups(rg).build(),
None => builder.set_row_groups(row_groups).build(),
})
}
- pub fn new(
+ pub(crate) fn new(
buf: &'a mut TrackedWrite<W>,
schema_descr: &'a SchemaDescPtr,
row_groups: Vec<RowGroupMetaData>,
created_by: Option<String>,
writer_version: i32,
write_path_in_schema: bool,
+ preserve_page_index_locations: bool,
) -> Self {
Self {
buf,
schema_descr,
row_groups,
- column_indexes: None,
- offset_indexes: None,
+ page_index: None,
key_value_metadata: None,
created_by,
object_writer: Default::default(),
writer_version,
write_path_in_schema,
+ preserve_page_index_locations,
}
}
- pub fn with_column_indexes(
- mut self,
- column_indexes: Vec<Vec<Option<ColumnIndexMetaData>>>,
- ) -> Self {
- self.column_indexes = Some(column_indexes);
- self
- }
-
- pub fn with_offset_indexes(
- mut self,
- offset_indexes: Vec<Vec<Option<OffsetIndexMetaData>>>,
- ) -> Self {
- self.offset_indexes = Some(offset_indexes);
+ pub(crate) fn with_page_index(mut self, page_index: Arc<dyn
PageIndexProvider>) -> Self {
+ self.page_index = Some(page_index);
self
}
- pub fn with_key_value_metadata(mut self, key_value_metadata:
Vec<KeyValue>) -> Self {
+ pub(crate) fn with_key_value_metadata(mut self, key_value_metadata:
Vec<KeyValue>) -> Self {
self.key_value_metadata = Some(key_value_metadata);
self
}
#[cfg(feature = "encryption")]
- pub fn with_file_encryptor(mut self, file_encryptor:
Option<Arc<FileEncryptor>>) -> Self {
+ pub(crate) fn with_file_encryptor(
+ mut self,
+ file_encryptor: Option<Arc<FileEncryptor>>,
+ ) -> Self {
self.object_writer =
self.object_writer.with_file_encryptor(file_encryptor);
self
}
@@ -333,16 +305,13 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> {
/// metadata writer. Then set the corresponding `bloom_filter_offset` and
/// `bloom_filter_length` on [`ColumnChunkMetaData`] passed to this writer.
///
-/// <div class="warning">
-///
-/// **NOTE:**
-/// The serialization of custom [`PageIndexProvider`]s is not currently
supported.
-/// The only supported page index structure is [`PageIndex`]. If the metadata
-/// contains any other [`PageIndexProvider`] implementation, the
[`ColumnIndex`]
-/// and [`OffsetIndex`] structures are silently omitted from the output. See
-/// <https://github.com/apache/arrow-rs/issues/11030> for more details.
-///
-/// </div>
+/// The writer serializes the page index of any [`PageIndexProvider`] attached
+/// to the input [`ParquetMetaData`]. By default, if the provider has no index
+/// for a column chunk, the offset and length of that index in the
[`ColumnChunkMetaData`]
+/// are preserved in the output metadata. This is done so one can store the
footer
+/// metadata externally but keep the page indexes in the original file. If this
+/// behavior is not desired, then [`Self::with_preserve_page_index_locations`]
should
+/// be set to `false`.
///
/// # Output Format
///
@@ -406,6 +375,7 @@ pub struct ParquetMetaDataWriter<'a, W: Write> {
buf: TrackedWrite<W>,
metadata: &'a ParquetMetaData,
write_path_in_schema: bool,
+ preserve_page_index_locations: bool,
}
impl<'a, W: Write> ParquetMetaDataWriter<'a, W> {
@@ -431,6 +401,7 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> {
buf,
metadata,
write_path_in_schema: true,
+ preserve_page_index_locations: true,
}
}
@@ -443,6 +414,23 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> {
}
}
+ /// Set whether or not to preserve the page index location metadata in the
Thrift
+ /// `ColumnMetaData` (defaults to `true`).
+ ///
+ /// Because this struct is often used to externalize the footer metadata,
it is
+ /// usually desirable to preserve this location information, even when the
+ /// page indexes are not duplicated (for instance if the provided
`ParquetMetaData`
+ /// returns no `PageIndexProvider`). As such, this defaults to `true`.
+ ///
+ /// Set this to `false` to reset the location metadata if no page indexes
are
+ /// present in the provided [`ParquetMetaData`].
+ pub fn with_preserve_page_index_locations(self, val: bool) -> Self {
+ Self {
+ preserve_page_index_locations: val,
+ ..self
+ }
+ }
+
/// Write the metadata to the buffer
pub fn finish(mut self) -> Result<()> {
let file_metadata = self.metadata.file_metadata();
@@ -462,23 +450,11 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> {
created_by,
file_metadata.version(),
self.write_path_in_schema,
+ self.preserve_page_index_locations,
);
- // Downcast to PageIndex to access raw index structures for
serialization.
- // Page indexes from custom PageIndexProviders are not written. See
- // <https://github.com/apache/arrow-rs/issues/11030>
- if let Some(page_index_arc) = self.metadata.page_index.as_ref()
- && let Some(page_index) = page_index_arc
- .as_any()
- .downcast_ref::<crate::file::metadata::PageIndex>()
- {
- if let Some(column_indexes) = page_index.column_indexes_raw() {
- encoder = encoder.with_column_indexes(column_indexes.clone());
- }
-
- if let Some(offset_indexes) = page_index.offset_indexes_raw() {
- encoder = encoder.with_offset_indexes(offset_indexes.clone());
- }
+ if let Some(page_index_arc) = self.metadata.page_index.clone() {
+ encoder = encoder.with_page_index(page_index_arc);
}
if let Some(key_value_metadata) = key_value_metadata {
diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs
index 6c0f3e8dd2..af60326ca3 100644
--- a/parquet/src/file/writer.rs
+++ b/parquet/src/file/writer.rs
@@ -18,6 +18,7 @@
//! [`SerializedFileWriter`]: Low level Parquet writer API
use crate::bloom_filter::Sbbf;
+use crate::file::metadata::page_index::PageIndex;
use crate::file::metadata::thrift::PageHeader;
use crate::file::page_index::column_index::ColumnIndexMetaData;
use crate::file::page_index::offset_index::OffsetIndexMetaData;
@@ -366,6 +367,7 @@ impl<W: Write + Send> SerializedFileWriter<W> {
Some(self.props.created_by().to_string()),
self.props.writer_version().as_num(),
write_path_in_schema,
+ false,
);
#[cfg(feature = "encryption")]
@@ -377,10 +379,27 @@ impl<W: Write + Send> SerializedFileWriter<W> {
encoder = encoder.with_key_value_metadata(key_value_metadata)
}
- encoder = encoder.with_column_indexes(column_indexes);
- if !self.props.offset_index_disabled() {
- encoder = encoder.with_offset_indexes(offset_indexes);
+ // check for empty column index
+ let column_indexes = if column_indexes.is_empty()
+ || column_indexes
+ .iter()
+ .all(|cis| cis.iter().all(|ci| ci.is_none()))
+ {
+ None
+ } else {
+ Some(column_indexes)
+ };
+ // offset index will always be created unless explicitly disabled
+ let offset_indexes = if self.props.offset_index_disabled() {
+ None
+ } else {
+ Some(offset_indexes)
+ };
+ if column_indexes.is_some() || offset_indexes.is_some() {
+ let page_index = PageIndex::new(column_indexes, offset_indexes);
+ encoder = encoder.with_page_index(Arc::new(page_index));
}
+
encoder.finish()
}
@@ -2201,6 +2220,55 @@ mod tests {
assert!(b_idx.is_none(), "{b_idx:?}");
}
+ #[test]
+ fn test_offset_index_disabled() {
+ let message_type = "
+ message test_schema {
+ REQUIRED INT32 a;
+ REQUIRED INT32 b;
+ }
+ ";
+ // write file with indexes disabled (including offset indexes)
+ let schema = Arc::new(parse_message_type(message_type).unwrap());
+ let props = WriterProperties::builder()
+ .set_statistics_enabled(EnabledStatistics::None)
+ .set_offset_index_disabled(true)
+ .build();
+ let mut file = Vec::with_capacity(1024);
+ let mut file_writer =
+ SerializedFileWriter::new(&mut file, schema,
Arc::new(props)).unwrap();
+
+ let mut row_group_writer = file_writer.next_row_group().unwrap();
+ let mut a_writer = row_group_writer.next_column().unwrap().unwrap();
+ let col_writer = a_writer.typed::<Int32Type>();
+ col_writer.write_batch(&[1, 2, 3], None, None).unwrap();
+ a_writer.close().unwrap();
+
+ let mut b_writer = row_group_writer.next_column().unwrap().unwrap();
+ let col_writer = b_writer.typed::<Int32Type>();
+ col_writer.write_batch(&[4, 5, 6], None, None).unwrap();
+ b_writer.close().unwrap();
+ row_group_writer.close().unwrap();
+
+ let metadata = file_writer.finish().unwrap();
+ assert_eq!(metadata.num_row_groups(), 1);
+ let row_group = metadata.row_group(0);
+ assert_eq!(row_group.num_columns(), 2);
+ // no page indexes should exist
+ assert!(row_group.column(0).offset_index_offset().is_none());
+ assert!(row_group.column(0).column_index_offset().is_none());
+ assert!(row_group.column(1).offset_index_offset().is_none());
+ assert!(row_group.column(1).column_index_offset().is_none());
+
+ drop(file_writer);
+
+ // read file and request page index...should be `None`
+ let options = ReadOptionsBuilder::new().with_page_index().build();
+ let reader = SerializedFileReader::new_with_options(Bytes::from(file),
options).unwrap();
+
+ assert!(reader.metadata().page_index().is_none());
+ }
+
#[test]
fn test_byte_array_size_statistics() {
let message_type = "
