H-Plus-Time commented on code in PR #7334:
URL: https://github.com/apache/arrow-rs/pull/7334#discussion_r2015143645
##########
parquet/src/file/metadata/reader.rs:
##########
@@ -638,6 +676,57 @@ impl ParquetMetaDataReader {
}
}
+ #[cfg(all(feature = "async", feature = "arrow"))]
+ async fn load_metadata_via_suffix<F: MetadataSuffixFetch>(
+ &self,
+ fetch: &mut F,
+ prefetch: usize,
+ ) -> Result<(ParquetMetaData, Option<(usize, Bytes)>)> {
+ let suffix = fetch.fetch_suffix(prefetch).await?;
+ let suffix_len = suffix.len();
+
+ // TODO: not sure how to restore this error handling here
+
+ // let fetch_len = file_size - footer_start;
+ // if suffix_len < fetch_len {
+ // return Err(eof_err!(
+ // "metadata requires {} bytes, but could only read {}",
+ // fetch_len,
+ // suffix_len
+ // ));
+ // }
+
+ let mut footer = [0; FOOTER_SIZE];
+ footer.copy_from_slice(&suffix[suffix_len - FOOTER_SIZE..suffix_len]);
+
+ let footer = Self::decode_footer_tail(&footer)?;
+ let length = footer.metadata_length();
+
+ // TODO: not sure how to restore this error handling here
+
+ // if file_size < length + FOOTER_SIZE {
+ // return Err(eof_err!(
+ // "file size of {} is less than footer + metadata {}",
+ // file_size,
+ // length + FOOTER_SIZE
+ // ));
+ // }
+
+ // Did not fetch the entire file metadata in the initial read, need to
make a second request
+ let metadata_offset = length + FOOTER_SIZE;
+ if length > suffix_len - FOOTER_SIZE {
+ let meta = fetch.fetch_suffix(metadata_offset).await?;
+ Ok((self.decode_footer_metadata(&meta, &footer)?, None))
+ } else {
+ let metadata_start = suffix_len - metadata_offset;
+ let slice = &suffix[metadata_start..suffix_len - FOOTER_SIZE];
+ Ok((
+ self.decode_footer_metadata(slice, &footer)?,
+ Some((0, suffix.slice(..metadata_start))),
+ ))
+ }
Review Comment:
Thank you for picking up on this one, I may have (definitely) ran out of
steam :face_exhaling:.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
