tustvold commented on code in PR #2526:
URL: https://github.com/apache/arrow-rs/pull/2526#discussion_r950701675
##########
parquet/src/arrow/async_reader.rs:
##########
@@ -218,6 +221,96 @@ impl<T: AsyncFileReader + Send + 'static>
ArrowReaderBuilder<AsyncReader<T>> {
Self::new_builder(AsyncReader(input), metadata, Default::default())
}
+ pub async fn new_with_options(
+ mut input: T,
+ options: ArrowReaderOptions,
+ ) -> Result<Self> {
+ let mut metadata = input.get_metadata().await?;
+
+ if options.page_index
+ && metadata
+ .page_indexes()
+ .zip(metadata.offset_indexes())
+ .is_none()
+ {
+ let mut fetch_ranges = vec![];
+ let mut index_lengths: Vec<Vec<usize>> = vec![];
+
+ for rg in metadata.row_groups() {
+ let (loc_offset, loc_length) =
+
index_reader::get_location_offset_and_total_length(rg.columns())?;
+
+ let (idx_offset, idx_lengths) =
+ index_reader::get_index_offset_and_lengths(rg.columns())?;
+ let idx_length = idx_lengths.iter().sum::<usize>();
+
+ // If index data is missing, return without any indexes
+ if loc_length == 0 || idx_length == 0 {
+ return Self::new_builder(AsyncReader(input), metadata,
options);
+ }
+
+ fetch_ranges.push(loc_offset as usize..loc_offset as usize +
loc_length);
Review Comment:
The onus is on `AsyncFileReader::get_ranges` to handle this, e.g.
`ObjectStore::get_ranges` does this already
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
