This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-24037-30eccb44e9a7d069b2c0c4d27abe8f39590acba3 in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit c8665ce95ac3cb3ab3b40cd28095022d77a791d7 Author: Andrew Lamb <[email protected]> AuthorDate: Tue Aug 4 15:47:56 2026 -0400 docs: document all fields and methods of `DFParquetMetadata` (#24037) ## Which issue does this PR close? - Something I noticed while working on https://github.com/apache/datafusion/pull/24036 ## Rationale for this change `DFParquetMetadata` had several fields and methods with no doc comments, or only a one-line description, making it hard to understand their purpose without reading the implementation. ## What changes are included in this PR? Add doc comments ## Are these changes tested? No behavior changed, only doc comments were added; existing tests cover the code. ## Are there any user-facing changes? No functional changes; public API doc comments are improved. --- datafusion/datasource-parquet/src/metadata.rs | 85 ++++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 8 deletions(-) diff --git a/datafusion/datasource-parquet/src/metadata.rs b/datafusion/datasource-parquet/src/metadata.rs index 56abf52144..3294ee00f1 100644 --- a/datafusion/datasource-parquet/src/metadata.rs +++ b/datafusion/datasource-parquet/src/metadata.rs @@ -65,11 +65,26 @@ const PARTIAL_NDV_THRESHOLD: f64 = 0.75; /// [`ParquetFileReaderFactory`]: crate::ParquetFileReaderFactory #[derive(Debug)] pub struct DFParquetMetadata<'a> { + /// Source of the Parquet file's bytes. store: &'a dyn ObjectStore, + /// Location, size and last-modified time of the target Parquet file. object_meta: &'a ObjectMeta, + /// Hint for the number of trailing bytes to prefetch before parsing the + /// footer, mirroring [`ParquetMetaDataReader::with_prefetch_hint`]. metadata_size_hint: Option<usize>, + /// Decryption properties used to read files encrypted with Parquet + /// Modular Encryption, mirroring + /// [`ParquetMetaDataReader::with_decryption_properties`]. decryption_properties: Option<Arc<FileDecryptionProperties>>, + /// Optional cache of previously fetched [`ParquetMetaData`], keyed by + /// file location. file_metadata_cache: Option<Arc<FileMetadataCache>>, + /// Policy controlling whether the Parquet page index (column and offset + /// indexes) is fetched, mirroring + /// [`ParquetMetaDataReader::with_page_index_policy`]. + /// + /// `None` means the effective policy is chosen automatically, see + /// [`DFParquetMetadata::effective_page_index_policy`]. page_index_policy: Option<PageIndexPolicy>, /// timeunit to coerce INT96 timestamps to pub coerce_int96: Option<TimeUnit>, @@ -78,6 +93,10 @@ pub struct DFParquetMetadata<'a> { } impl<'a> DFParquetMetadata<'a> { + /// Create a new `DFParquetMetadata` for the given file. + /// + /// Use the `with_*` builder methods to customize behavior + /// before calling [`Self::fetch_metadata`] or [`Self::fetch_schema`]. pub fn new(store: &'a dyn ObjectStore, object_meta: &'a ObjectMeta) -> Self { Self { store, @@ -91,13 +110,23 @@ impl<'a> DFParquetMetadata<'a> { } } - /// set metadata size hint + /// Set a hint for the number of trailing bytes to prefetch from the end + /// of the file, equivalent to + /// [`ParquetMetaDataReader::with_prefetch_hint`]. + /// + /// Providing a good estimate of the footer (and, if requested, page index) + /// size can save an extra I/O round trip when fetching metadata from the + /// store. pub fn with_metadata_size_hint(mut self, metadata_size_hint: Option<usize>) -> Self { self.metadata_size_hint = metadata_size_hint; self } - /// set decryption properties + /// Set the decryption properties used to read an encrypted Parquet file, + /// equivalent to [`ParquetMetaDataReader::with_decryption_properties`]. + /// + /// Only needed when the target file was written with Parquet Modular + /// Encryption. pub fn with_decryption_properties( mut self, decryption_properties: Option<Arc<FileDecryptionProperties>>, @@ -106,7 +135,8 @@ impl<'a> DFParquetMetadata<'a> { self } - /// set file metadata cache + /// Set an optional [`FileMetadataCache`] used to avoid re-fetching + /// [`ParquetMetaData`] for files that have already been read. pub fn with_file_metadata_cache( mut self, file_metadata_cache: Option<Arc<FileMetadataCache>>, @@ -115,7 +145,12 @@ impl<'a> DFParquetMetadata<'a> { self } - /// Sets the policy for loading parquet page index structures (column and offset indexes). + /// Sets the policy for loading parquet page index structures (column and + /// offset indexes), equivalent to + /// [`ParquetMetaDataReader::with_page_index_policy`]. + /// + /// Passing `None` uses a default automatically, based on whether a metadata + /// cache is configured. pub fn with_page_index_policy( mut self, page_index_policy: Option<PageIndexPolicy>, @@ -124,19 +159,31 @@ impl<'a> DFParquetMetadata<'a> { self } - /// Set timeunit to coerce INT96 timestamps to + /// Set the [`TimeUnit`] that INT96 timestamp columns should be coerced + /// to when reading the schema. + /// + /// INT96 in Parquet has no defined unit or timezone, so leaving this + /// `None` reads INT96 columns as nanosecond timestamps with no timezone + /// — DataFusion's default behavior. pub fn with_coerce_int96(mut self, time_unit: Option<TimeUnit>) -> Self { self.coerce_int96 = time_unit; self } /// Set the optional timezone applied to INT96-coerced timestamps. + /// + /// Only used when [`Self::with_coerce_int96`] has also been set, and + /// otherwise has no effect. pub fn with_coerce_int96_tz(mut self, timezone: Option<Arc<str>>) -> Self { self.coerce_int96_tz = timezone; self } - /// Fetch parquet metadata from the remote object store + /// Fetch the [`ParquetMetaData`] for this file. + /// + /// Consults the [`FileMetadataCache`] first when one is configured and + /// falls back to reading from the object store via + /// [`ParquetMetaDataPushDecoder`] on a cache miss. pub async fn fetch_metadata(&self) -> Result<Arc<ParquetMetaData>> { // fetch_metadata // │ @@ -193,9 +240,15 @@ impl<'a> DFParquetMetadata<'a> { Ok(metadata) } + /// Resolve the [`PageIndexPolicy`] to use for a fetch. fn effective_page_index_policy(&self, cache_metadata: bool) -> PageIndexPolicy { self.page_index_policy.unwrap_or_else(|| { + // fetching the page index often requires a second IO (after the + // main metadata), so it is not free. if cache_metadata && self.file_metadata_cache.is_some() { + // When there is a cache available, retrieve the page index + // heuristically on the assumption it will be used multiple + // times PageIndexPolicy::Optional } else { PageIndexPolicy::Skip @@ -203,10 +256,20 @@ impl<'a> DFParquetMetadata<'a> { }) } + /// Check whether `metadata` already has both the column index and the + /// offset index populated (see [`ParquetMetaData::column_index`] and + /// [`ParquetMetaData::offset_index`]). + /// + /// Used to decide whether page index I/O can be skipped. fn metadata_has_page_index(metadata: &ParquetMetaData) -> bool { metadata.column_index().is_some() && metadata.offset_index().is_some() } + /// Store `metadata` in the configured [`FileMetadataCache`], keyed by + /// the file's location. + /// + /// This is a no-op unless a cache has been configured via + /// [`Self::with_file_metadata_cache`]. fn cache_metadata(&self, metadata: Arc<ParquetMetaData>) -> Result<()> { if let Some(file_metadata_cache) = &self.file_metadata_cache { file_metadata_cache.put( @@ -220,6 +283,8 @@ impl<'a> DFParquetMetadata<'a> { Ok(()) } + /// Fetch the full [`ParquetMetaData`] (including footer, and optional + /// page index) from the object store. async fn fetch_metadata_from_store( &self, page_index_policy: PageIndexPolicy, @@ -277,6 +342,8 @@ impl<'a> DFParquetMetadata<'a> { Ok(Arc::new(metadata)) } + /// If `metadata` does not already have a page index, fetch and attach the + /// column and offset indexes. async fn load_page_index( store: &dyn ObjectStore, object_meta: &ObjectMeta, @@ -297,7 +364,8 @@ impl<'a> DFParquetMetadata<'a> { Ok(Arc::new(reader.finish().map_err(DataFusionError::from)?)) } - /// Read and parse the schema of the Parquet file + /// Fetch this file's [`ParquetMetaData`] and convert its embedded Thrift + /// schema into an Arrow [`Schema`]. pub async fn fetch_schema(&self) -> Result<Schema> { let metadata = self.fetch_metadata().await?; @@ -318,7 +386,8 @@ impl<'a> DFParquetMetadata<'a> { Ok(schema) } - /// Return (path, schema) tuple by fetching the schema from Parquet file + /// Convenience wrapper around [`Self::fetch_schema`] that also returns + /// the file's object store [`Path`]. pub(crate) async fn fetch_schema_with_location(&self) -> Result<(Path, Schema)> { let loc_path = self.object_meta.location.clone(); let schema = self.fetch_schema().await?; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
