tustvold commented on code in PR #2526:
URL: https://github.com/apache/arrow-rs/pull/2526#discussion_r950701280
##########
parquet/src/arrow/async_reader.rs:
##########
@@ -218,6 +221,96 @@ impl<T: AsyncFileReader + Send + 'static>
ArrowReaderBuilder<AsyncReader<T>> {
Self::new_builder(AsyncReader(input), metadata, Default::default())
}
+ pub async fn new_with_options(
+ mut input: T,
+ options: ArrowReaderOptions,
+ ) -> Result<Self> {
+ let mut metadata = input.get_metadata().await?;
+
+ if options.page_index
+ && metadata
+ .page_indexes()
+ .zip(metadata.offset_indexes())
+ .is_none()
+ {
+ let mut fetch_ranges = vec![];
+ let mut index_lengths: Vec<Vec<usize>> = vec![];
+
+ for rg in metadata.row_groups() {
+ let (loc_offset, loc_length) =
+
index_reader::get_location_offset_and_total_length(rg.columns())?;
+
+ let (idx_offset, idx_lengths) =
+ index_reader::get_index_offset_and_lengths(rg.columns())?;
+ let idx_length = idx_lengths.iter().sum::<usize>();
+
+ // If index data is missing, return without any indexes
+ if loc_length == 0 || idx_length == 0 {
+ return Self::new_builder(AsyncReader(input), metadata,
options);
+ }
+
+ fetch_ranges.push(loc_offset as usize..loc_offset as usize +
loc_length);
+ fetch_ranges.push(idx_offset as usize..idx_offset as usize +
idx_length);
+ index_lengths.push(idx_lengths);
+ }
+
+ let mut chunks =
input.get_byte_ranges(fetch_ranges).await?.into_iter();
+ let mut index_lengths = index_lengths.into_iter();
+
+ let mut row_groups = metadata.row_groups().to_vec();
+
+ let mut columns_indexes = vec![];
+ let mut offset_indexes = vec![];
+
+ for rg in row_groups.iter_mut() {
+ let columns = rg.columns();
+
+ let location_data = chunks.next().unwrap();
+ let mut cursor = Cursor::new(location_data);
+ let mut offset_index = vec![];
+
+ for _ in 0..columns.len() {
+ let mut prot = TCompactInputProtocol::new(&mut cursor);
+ let offset = OffsetIndex::read_from_in_protocol(&mut
prot)?;
+ offset_index.push(offset.page_locations);
+ }
+
+ rg.set_page_offset(offset_index.clone());
+ offset_indexes.push(offset_index);
Review Comment:
Again this interface seems really confused - something for #2530
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
