Rich-T-kid commented on code in PR #27500:
URL: https://github.com/apache/datafusion/pull/27500#discussion_r4656804588
##########
datafusion/datasource-parquet/src/metadata.rs:
##########
@@ -39,13 +40,74 @@ use parquet::file::metadata::{
use parquet::file::statistics::Statistics as ParquetStatistics;
use parquet::schema::types::{ColumnDescriptor, SchemaDescriptor};
use std::any::Any;
+use std::collections::HashSet;
use std::sync::Arc;
/// Minimum fraction of row groups that must report NDV statistics for the
/// merged result to be `Inexact` rather than `Absent`, as the estimate
/// would be too unreliable otherwise.
const PARTIAL_NDV_THRESHOLD: f64 = 0.75;
+fn dictionary_promotion_candidates(
+ metadata: &ParquetMetaData,
+ schema_descr: &SchemaDescriptor,
+ max_distinct_values: Option<usize>,
+) -> HashSet<String> {
+ (0..schema_descr.num_columns())
+ .filter_map(|col_idx| {
+ let column = schema_descr.column(col_idx);
+ let parts = column.path().parts();
+ if parts.len() != 1 {
+ return None;
+ }
+
+ let row_groups = metadata.row_groups();
+ let eligible = match max_distinct_values {
+ None => row_groups
+ .iter()
+ .any(|rg|
rg.column(col_idx).dictionary_page_offset().is_some()),
+ Some(max) => row_groups
+ .iter()
+ .try_fold(0_u128, |upper_bound, row_group| {
+ let row_count =
u64::try_from(row_group.num_rows()).ok()?;
+ let column = row_group.column(col_idx);
+ // A chunk whose dictionary outgrew the writer's page
limit
+ // continues in PLAIN pages, and reads slower as a
dictionary
+ // than as plain values.
+ let all_dictionary_pages = column
+ .dictionary_page_offset()
+ .is_some()
+ &&
column.page_encoding_stats_mask().is_some_and(|mask| {
+
mask.is_only(parquet::basic::Encoding::RLE_DICTIONARY)
+ || mask.is_only(
+
parquet::basic::Encoding::PLAIN_DICTIONARY,
Review Comment:
updated this as well
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]