tustvold commented on code in PR #2526:
URL: https://github.com/apache/arrow-rs/pull/2526#discussion_r950700929
##########
parquet/src/arrow/async_reader.rs:
##########
@@ -218,6 +221,96 @@ impl<T: AsyncFileReader + Send + 'static>
ArrowReaderBuilder<AsyncReader<T>> {
Self::new_builder(AsyncReader(input), metadata, Default::default())
}
+ pub async fn new_with_options(
+ mut input: T,
+ options: ArrowReaderOptions,
+ ) -> Result<Self> {
+ let mut metadata = input.get_metadata().await?;
+
+ if options.page_index
+ && metadata
+ .page_indexes()
+ .zip(metadata.offset_indexes())
+ .is_none()
+ {
+ let mut fetch_ranges = vec![];
+ let mut index_lengths: Vec<Vec<usize>> = vec![];
+
+ for rg in metadata.row_groups() {
+ let (loc_offset, loc_length) =
+
index_reader::get_location_offset_and_total_length(rg.columns())?;
+
+ let (idx_offset, idx_lengths) =
+ index_reader::get_index_offset_and_lengths(rg.columns())?;
+ let idx_length = idx_lengths.iter().sum::<usize>();
+
+ // If index data is missing, return without any indexes
+ if loc_length == 0 || idx_length == 0 {
+ return Self::new_builder(AsyncReader(input), metadata,
options);
+ }
+
+ fetch_ranges.push(loc_offset as usize..loc_offset as usize +
loc_length);
Review Comment:
This can't use read_columns_indexes as this needs to asynchronously fetch
the bytes
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
