leaves12138 commented on code in PR #1041:
URL: https://github.com/apache/paimon-rust/pull/1041#discussion_r4184313050
##########
crates/paimon/src/arrow/format/parquet.rs:
##########
@@ -1060,29 +1050,82 @@ impl FormatFileReader for ParquetFormatReader {
}
}
-fn map_shredding_projection_mask(
+fn read_type_projection_mask(
parquet_schema: &parquet::schema::types::SchemaDescriptor,
- selected_roots: &[usize],
- plan: &MapShreddingReadPlan,
+ read_fields: &[DataField],
+ map_plan: Option<&MapShreddingReadPlan>,
+ predicates: &[Predicate],
) -> ProjectionMask {
- let selected_roots: std::collections::HashSet<usize> =
selected_roots.iter().copied().collect();
- let root_fields = parquet_schema.root_schema().get_fields();
- let leaves = (0..parquet_schema.num_columns()).filter(|leaf_index| {
- let root_index = parquet_schema.get_column_root_idx(*leaf_index);
- if !selected_roots.contains(&root_index) {
- return false;
+ fn requested_paths(
+ data_type: &DataType,
+ physical: &parquet::schema::types::Type,
+ path: &mut Vec<String>,
+ output: &mut std::collections::HashSet<Vec<String>>,
+ ) {
+ if let DataType::Row(row) = data_type {
+ if crate::spec::is_variant_extraction_row_type(data_type) {
+ output.extend(
+ super::variant_projection::projected_paths(row, physical)
+ .into_iter()
+ .map(|parts| {
+ let mut full = path.clone();
+ full.extend(parts);
+ full
+ }),
+ );
+ return;
+ }
+ if !physical.is_primitive() {
+ for field in row.fields() {
+ if let Some(child) = physical
+ .get_fields()
+ .iter()
+ .find(|child| child.name() == field.name())
+ {
+ path.push(child.name().to_string());
+ requested_paths(field.data_type(), child, path,
output);
+ path.pop();
+ }
+ }
+ return;
+ }
}
- let root_name = root_fields[root_index].name();
- let Some(children) = plan.projected_physical_children(root_name) else {
- return true;
+ super::variant_projection::all_leaves(physical, path, output);
+ }
+ let mut predicate_refs = Vec::new();
+ for predicate in predicates {
+ crate::arrow::residual::collect_predicate_leaf_refs(predicate, &mut
predicate_refs);
+ }
+ let roots = parquet_schema.root_schema().get_fields();
+ let mut selected = std::collections::HashSet::new();
+ for field in read_fields {
+ let Some(root) = roots.iter().find(|root| root.name() == field.name())
else {
+ continue;
};
- parquet_schema
- .column(*leaf_index)
- .path()
- .parts()
- .get(1)
- .is_some_and(|child| children.contains(child))
- });
+ let mut path = vec![root.name().to_string()];
+ if predicate_refs
+ .iter()
+ .any(|(column, _)| *column == field.name())
+ {
+ super::variant_projection::all_leaves(root, &mut path, &mut
selected);
+ } else if let Some(children) =
+ map_plan.and_then(|plan|
plan.projected_physical_children(field.name()))
+ {
+ for child in root
+ .get_fields()
+ .iter()
+ .filter(|child| children.contains(child.name()))
+ {
+ path.push(child.name().to_string());
+ super::variant_projection::all_leaves(child, &mut path, &mut
selected);
+ path.pop();
+ }
+ } else {
+ requested_paths(field.data_type(), root, &mut path, &mut selected);
Review Comment:
This does not match Java's selected-key MAP contract.
`validate_selected_map_fields` currently accepts a synthetic selected-key ROW
for any STRING-keyed MAP (and recursively accepts nested MAPs), but
`MapShreddingReadPlan` only creates a context for top-level shared-shredding
MAPs. For an ordinary MAP, `map_plan` is `None` here, so `requested_paths`
compares the selected key names with the physical MAP group children
(`key_value`/`key`/`value`), produces no leaves, and the reader can decode an
empty projection instead of the requested values. Java's
`FormatReaderMapping#selectedKeysFieldIds` rejects this shape unless the source
is a top-level shared-shredding MAP. Please either enforce the same validation
before reaching this path or add a complete-map fallback/assembly for
non-shredded and nested MAPs.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
