alamb commented on code in PR #6630:
URL: https://github.com/apache/arrow-rs/pull/6630#discussion_r1820736077
##########
parquet/src/file/metadata/reader.rs:
##########
@@ -192,18 +194,54 @@ impl ParquetMetaDataReader {
/// # fn open_parquet_file(path: &str) -> std::fs::File {
unimplemented!(); }
/// let file = open_parquet_file("some_path.parquet");
/// let len = file.len() as usize;
- /// let bytes = get_bytes(&file, 1000..len);
+ /// // Speculatively read 1 kilobyte from the end of the file
+ /// let bytes = get_bytes(&file, len - 1024..len);
/// let mut reader = ParquetMetaDataReader::new().with_page_indexes(true);
/// match reader.try_parse_sized(&bytes, len) {
/// Ok(_) => (),
/// Err(ParquetError::NeedMoreData(needed)) => {
+ /// // Read the needed number of bytes from the end of the file
/// let bytes = get_bytes(&file, len - needed..len);
/// reader.try_parse_sized(&bytes, len).unwrap();
/// }
/// _ => panic!("unexpected error")
/// }
/// let metadata = reader.finish().unwrap();
/// ```
+ ///
+ /// Note that it is possible for the file metadata to be completely read,
but there are
+ /// insufficient bytes available to read the page indexes.
[`Self::has_metadata()`] can be used
+ /// to test for this. In the event the file metadata is present,
re-parsing of the file
+ /// metadata can be skipped by using [`Self::read_page_indexes_sized()`],
as shown below.
+ /// ```no_run
+ /// # use parquet::file::metadata::ParquetMetaDataReader;
+ /// # use parquet::errors::ParquetError;
+ /// # use crate::parquet::file::reader::Length;
+ /// # fn get_bytes(file: &std::fs::File, range: std::ops::Range<usize>) ->
bytes::Bytes { unimplemented!(); }
+ /// # fn open_parquet_file(path: &str) -> std::fs::File {
unimplemented!(); }
+ /// let file = open_parquet_file("some_path.parquet");
+ /// let len = file.len() as usize;
+ /// // Speculatively read 1 kilobyte from the end of the file
+ /// let mut bytes = get_bytes(&file, len - 1024..len);
+ /// let mut reader = ParquetMetaDataReader::new().with_page_indexes(true);
+ /// // Loop until `bytes` is large enough
+ /// loop {
+ /// match reader.try_parse_sized(&bytes, len) {
+ /// Ok(_) => break,
+ /// Err(ParquetError::NeedMoreData(needed)) => {
+ /// // Read the needed number of bytes from the end of the file
+ /// bytes = get_bytes(&file, len - needed..len);
Review Comment:
👍
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
