Copilot commented on code in PR #3658:
URL: https://github.com/apache/iceberg-rust/pull/3658#discussion_r4557365454
##########
crates/iceberg/src/arrow/reader/projection.rs:
##########
@@ -545,6 +543,174 @@ impl PresenceWalk<'_> {
}
}
+/// Fails on a field id that is missing or repeated, as
`arrow_schema_to_schema` would.
+fn check_field_ids<'a>(
+ fields: impl Iterator<Item = &'a FieldRef>,
+ seen: &mut HashSet<i32>,
+) -> Result<()> {
+ for field in fields {
+ let field_id = parquet_field_id(field)
+ .ok_or_else(|| invalid_data!("Field id not found in metadata"))?;
+ if !seen.insert(field_id) {
+ return Err(invalid_data!(
+ "Found duplicate 'field.id' {field_id}. Field ids must be
unique."
+ ));
+ }
+ check_child_field_ids(field.data_type(), seen)?;
+ }
+ Ok(())
+}
+
+fn check_child_field_ids(data_type: &DataType, seen: &mut HashSet<i32>) ->
Result<()> {
+ match data_type {
+ DataType::Struct(fields) => check_field_ids(fields.iter(), seen),
+ DataType::List(element)
+ | DataType::LargeList(element)
+ | DataType::FixedSizeList(element, _) =>
check_field_ids(std::iter::once(element), seen),
+ // The entries struct carries no field id.
+ DataType::Map(entries, _) =>
check_child_field_ids(entries.data_type(), seen),
+ DataType::Dictionary(_, value) => check_child_field_ids(value, seen),
+ DataType::RunEndEncoded(_, value) =>
check_child_field_ids(value.data_type(), seen),
+ _ => Ok(()),
+ }
+}
+
+/// Number of Parquet leaves under `data_type`, counted as
[`Fields::filter_leaves`] does.
+fn arrow_leaf_count(data_type: &DataType) -> usize {
+ match data_type {
+ DataType::Dictionary(_, value) => arrow_leaf_count(value),
+ DataType::RunEndEncoded(_, value) =>
arrow_leaf_count(value.data_type()),
+ DataType::List(child)
+ | DataType::LargeList(child)
+ | DataType::FixedSizeList(child, _)
+ | DataType::Map(child, _) => arrow_leaf_count(child.data_type()),
+ DataType::Struct(fields) => fields.iter().map(|f|
arrow_leaf_count(f.data_type())).sum(),
+ DataType::Union(fields, _) => fields
+ .iter()
+ .map(|(_, f)| arrow_leaf_count(f.data_type()))
+ .sum(),
+ _ => 1,
+ }
+}
+
+/// Leaf index ranges of one map's key and value.
+struct MapLeafRanges {
+ key: Range<usize>,
+ value: Range<usize>,
+ /// Value subtrees that match no table field and so read as NULL; `None`
unless the value
+ /// is a struct, as other values match the table by position.
+ null_subtrees: Option<Vec<Range<usize>>>,
+}
+
+/// Collects map leaf ranges in pre-order, so an outer map precedes the maps
in its value.
+fn collect_map_leaf_ranges(
+ data_type: &DataType,
+ next_leaf: &mut usize,
+ table_schema: &Schema,
+ maps: &mut Vec<MapLeafRanges>,
+) {
+ let children: Vec<&DataType> = match data_type {
+ DataType::Dictionary(_, value) => vec![value],
+ DataType::RunEndEncoded(_, value) => vec![value.data_type()],
+ DataType::List(child) | DataType::LargeList(child) |
DataType::FixedSizeList(child, _) => {
+ vec![child.data_type()]
+ }
+ DataType::Map(entries, _) => match entries.data_type() {
+ DataType::Struct(entry_fields) if entry_fields.len() == 2 => {
+ let (key, value) = (&entry_fields[0], &entry_fields[1]);
+ let value_start = *next_leaf +
arrow_leaf_count(key.data_type());
+ let value_end = value_start +
arrow_leaf_count(value.data_type());
+ let null_subtrees = match value.data_type() {
+ DataType::Struct(fields) => {
+ let mut subtrees = Vec::new();
+ collect_null_subtrees(fields, value_start,
table_schema, &mut subtrees);
+ Some(subtrees)
+ }
+ _ => None,
+ };
+ maps.push(MapLeafRanges {
+ key: *next_leaf..value_start,
+ value: value_start..value_end,
+ null_subtrees,
+ });
+ vec![key.data_type(), value.data_type()]
+ }
+ other => vec![other],
+ },
+ DataType::Struct(fields) => fields.iter().map(|f|
f.data_type()).collect(),
+ DataType::Union(fields, _) => fields.iter().map(|(_, f)|
f.data_type()).collect(),
+ _ => {
+ *next_leaf += 1;
+ return;
+ }
+ };
+ for child in children {
+ collect_map_leaf_ranges(child, next_leaf, table_schema, maps);
+ }
+}
+
+/// Collects the leaf ranges of struct fields whose id is not in the table:
the reader matches
+/// struct fields by id and drops them. Fields under a list or map match by
position instead.
+fn collect_null_subtrees(
+ fields: &Fields,
+ start: usize,
+ table_schema: &Schema,
+ subtrees: &mut Vec<Range<usize>>,
+) {
+ let mut next = start;
+ for field in fields {
+ let range = next..next + arrow_leaf_count(field.data_type());
+ next = range.end;
+ match parquet_field_id(field) {
+ Some(field_id) if table_schema.field_by_id(field_id).is_some() => {
+ if let DataType::Struct(children) = field.data_type() {
+ collect_null_subtrees(children, range.start, table_schema,
subtrees);
+ }
+ }
+ _ if !range.is_empty() => subtrees.push(range),
+ _ => {}
+ }
+ }
+}
+
+/// Parquet reads a map only when both its key and value are projected. When
the key is selected
+/// but no value leaf matched the table (#3557), also read the smallest value
subtree that matches
+/// no table field, so the value comes back as a struct of NULLs (#3437). A
struct field whose id
+/// is in the table is never read this way: its leaves were either rejected by
type promotion or
+/// sit under a list or map, which match by position and would be cast.
+fn add_map_value_leaves(
+ parquet_schema: &SchemaDescriptor,
+ arrow_fields: &Fields,
+ table_schema: &Schema,
+ indices: &mut Vec<usize>,
+) -> Result<()> {
+ let mut maps = Vec::new();
+ let mut next_leaf = 0;
+ for field in arrow_fields {
+ collect_map_leaf_ranges(field.data_type(), &mut next_leaf,
table_schema, &mut maps);
+ }
+ let mut selected: HashSet<usize> = indices.iter().copied().collect();
+ for map in &maps {
+ let key_read = map.key.clone().any(|idx| selected.contains(&idx));
+ let value_read = map.value.clone().any(|idx| selected.contains(&idx));
+ let Some(null_subtrees) = &map.null_subtrees else {
+ continue;
+ };
+ if !key_read || value_read {
+ continue;
+ }
+ let Some(subtree) = null_subtrees.iter().min_by_key(|range|
range.len()) else {
Review Comment:
For a name-mapped map value with matching nested structs `a` and `b` but
unmapped children `a.old_x` and `b.old_y`, choosing one subtree preserves only
`a`'s presence. The transformer returns NULL for a present `b`, or errors if
`b` is required. Select a discarded presence leaf beneath every projected
nested struct in the rescued value, and add a regression covering both structs.
##########
crates/iceberg/src/arrow/reader/projection.rs:
##########
@@ -545,6 +543,174 @@ impl PresenceWalk<'_> {
}
}
+/// Fails on a field id that is missing or repeated, as
`arrow_schema_to_schema` would.
+fn check_field_ids<'a>(
+ fields: impl Iterator<Item = &'a FieldRef>,
+ seen: &mut HashSet<i32>,
+) -> Result<()> {
+ for field in fields {
+ let field_id = parquet_field_id(field)
+ .ok_or_else(|| invalid_data!("Field id not found in metadata"))?;
+ if !seen.insert(field_id) {
+ return Err(invalid_data!(
+ "Found duplicate 'field.id' {field_id}. Field ids must be
unique."
+ ));
+ }
+ check_child_field_ids(field.data_type(), seen)?;
Review Comment:
Unlike `arrow_schema_to_schema`, this descends into `arrow.parquet.variant`
storage fields, which intentionally have no field IDs
(`arrow/schema.rs:678–689`). A projected struct whose presence candidate is a
dropped `map<string, variant>` now fails with "Field id not found in metadata",
even though the current table schema contains no variant. Validate the
enclosing variant ID without visiting its storage children, and add a
regression for this presence-only case.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]