etseidl commented on code in PR #11438:
URL: https://github.com/apache/arrow-rs/pull/11438#discussion_r4267249740
##########
parquet/src/file/metadata/push_decoder.rs:
##########
@@ -619,6 +700,66 @@ mod tests {
assert!(masked.end - masked.start < all.end - all.start);
}
+ #[test]
+ fn test_page_index_decoder_incremental() {
+ let metadata = test_metadata_without_page_index();
+ let mut decoder =
+ ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(),
metadata)
+ .unwrap()
+ .with_column_index_policy(PageIndexPolicy::Required)
+ .with_offset_index_policy(PageIndexPolicy::Required)
+
.with_column_index_mask(ColumnChunkMask::row_groups_and_columns([1], [0, 2]))
+
.with_offset_index_mask(ColumnChunkMask::row_groups_and_columns([0], [1]));
+
+ let ranges = expect_needs_data(decoder.try_decode_page_index());
+ assert_eq!(ranges.len(), 1);
+ push_ranges_to_metadata_decoder(&mut decoder, ranges);
+
+ let page_index = expect_data(decoder.try_decode_page_index()).unwrap();
+ expect_finished(decoder.try_decode_page_index());
+
+ for row_group_idx in 0..2 {
+ for column_idx in 0..3 {
+ assert_eq!(
+ page_index.column_index(row_group_idx,
column_idx).is_some(),
+ row_group_idx == 1 && matches!(column_idx, 0 | 2)
+ );
+ assert_eq!(
+ page_index.offset_index(row_group_idx,
column_idx).is_some(),
+ row_group_idx == 0 && column_idx == 1
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn test_page_index_decoder_no_indexes_requested() {
+ let metadata = test_metadata_without_page_index();
+ let mut decoder =
+ ParquetMetaDataPushDecoder::try_new_with_metadata(test_file_len(),
metadata)
+ .unwrap()
+ .with_page_index_policy(PageIndexPolicy::Skip);
+
+ assert!(expect_data(decoder.try_decode_page_index()).is_none());
+ expect_finished(decoder.try_decode_page_index());
+
+ let metadata = test_metadata_without_page_index();
Review Comment:
Finally looping back to this one. The preceding test
(`test_decode_page_index_after_metadata`) does what I think you're requesting
here. It first reads the metadata without a page index, then does a subsequent
call to the new `try_decode_page_index` with a mask set. Or am I
misunderstanding your intent?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
