Rich-T-kid commented on code in PR #24227:
URL: https://github.com/apache/datafusion/pull/24227#discussion_r3793151424
##########
datafusion/datasource-parquet/src/metadata.rs:
##########
@@ -384,6 +395,74 @@ impl<'a> DFParquetMetadata<'a> {
.coerce()
})
.unwrap_or(schema);
+
+ // Promote before force_view_types so dictionary columns survive the
+ // Utf8 -> Utf8View pass.
+ let schema = if self.enable_rle_to_dictionary {
+ let rle_cols: HashSet<String> = {
+ let schema_descr = file_metadata.schema_descr();
+ metadata
+ .row_groups()
+ .iter()
+ .flat_map(|rg| {
+ rg.columns()
+ .iter()
+ .enumerate()
+ .filter_map(|(col_idx, col)| {
+ col.dictionary_page_offset()?;
+ let desc = schema_descr.column(col_idx);
+ // Only promote top-level columns. For nested
columns
+ // the Parquet leaf name (e.g. "element"
inside a List)
+ // does not match the Arrow top-level field
name (e.g.
+ // "tags"), so using the leaf name here would
miss the
+ // target field or promote an unrelated one.
+ let parts = desc.path().parts();
+ (parts.len() == 1).then(|| parts[0].clone())
+ })
+ })
+ .collect()
+ };
+ if rle_cols.is_empty() {
+ schema
+ } else {
+ let promoted: Vec<_> = schema
+ .fields()
+ .iter()
+ .map(|field| {
+ if !rle_cols.contains(field.name()) {
+ return Arc::clone(field);
+ }
+ let dict_value_type = match field.data_type() {
+ DataType::Utf8 => Some(DataType::Utf8),
+ DataType::LargeUtf8 => Some(DataType::LargeUtf8),
+ DataType::Binary => Some(DataType::Binary),
+ DataType::LargeBinary =>
Some(DataType::LargeBinary),
+ _ => None,
+ };
+ dict_value_type.map_or_else(
+ || Arc::clone(field),
+ |value_type| {
+ Arc::new(
+ arrow::datatypes::Field::new(
Review Comment:
nit remove this path prefix
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
