This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-3396-9092073f988b3e6401d4ca00e2b50b7a53434ee1 in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
commit d3e9f456e29b10ba95f11b3a2bc5c9dc546e1d93 Author: Oleks V <[email protected]> AuthorDate: Sun Oct 11 18:28:33 2026 +0000 perf(reader): read leaf field IDs from metadata in the projection mask (#3396) get_arrow_projection_mask_with_field_ids carried each leaf's field ID from its first pass to its second through a HashMap<FieldRef, i32>. Hashing an Arrow Field hashes its whole DataType and allocates a sorted copy of its metadata keys, and the map grew from empty. Read the ID from the leaf's PARQUET:field_id metadata in both passes instead. Closes #3394 --- crates/iceberg/src/arrow/reader/projection.rs | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/projection.rs b/crates/iceberg/src/arrow/reader/projection.rs index fcb1b2bfd..8f2afe79e 100644 --- a/crates/iceberg/src/arrow/reader/projection.rs +++ b/crates/iceberg/src/arrow/reader/projection.rs @@ -178,23 +178,16 @@ impl ArrowReader { // Pre-project only the fields that have been selected, possibly avoiding converting // some Arrow types that are not yet supported. - let mut projected_fields: HashMap<arrow_schema::FieldRef, i32> = HashMap::new(); let projected_arrow_schema = ArrowSchema::new_with_metadata( fields.filter_leaves(|_, f| { - f.metadata() - .get(PARQUET_FIELD_ID_META_KEY) - .and_then(|field_id| i32::from_str(field_id).ok()) - .is_some_and(|field_id| { - projected_fields.insert((*f).clone(), field_id); - leaf_field_id_set.contains(&field_id) - }) + parquet_field_id(f).is_some_and(|field_id| leaf_field_id_set.contains(&field_id)) }), arrow_schema.metadata().clone(), ); let iceberg_schema = arrow_schema_to_schema(&projected_arrow_schema)?; fields.filter_leaves(|idx, field| { - let Some(field_id) = projected_fields.get(field).cloned() else { + let Some(field_id) = parquet_field_id(field) else { return false; }; @@ -381,11 +374,7 @@ pub(super) fn build_fallback_field_id_map( fn build_field_id_map_from_arrow_schema(arrow_schema: &ArrowSchemaRef) -> HashMap<i32, usize> { let mut column_map = HashMap::new(); arrow_schema.fields().filter_leaves(|idx, field| { - if let Some(field_id) = field - .metadata() - .get(PARQUET_FIELD_ID_META_KEY) - .and_then(|value| i32::from_str(value).ok()) - { + if let Some(field_id) = parquet_field_id(field) { column_map.insert(field_id, idx); } false @@ -393,6 +382,14 @@ fn build_field_id_map_from_arrow_schema(arrow_schema: &ArrowSchemaRef) -> HashMa column_map } +/// Returns the field ID in `field`'s `PARQUET:field_id` metadata, if it has a valid one. +fn parquet_field_id(field: &Field) -> Option<i32> { + field + .metadata() + .get(PARQUET_FIELD_ID_META_KEY) + .and_then(|field_id| i32::from_str(field_id).ok()) +} + /// Apply name mapping to Arrow schema for Parquet files lacking field IDs. /// /// Assigns Iceberg field IDs based on column names using the name mapping,
