parthchandra commented on code in PR #2668: URL: https://github.com/apache/iceberg-rust/pull/2668#discussion_r3502355383
########## crates/iceberg/src/partitioning.rs: ########## @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Partition type utilities for Iceberg tables. + +use crate::spec::{NestedField, NestedFieldRef, PartitionSpec, Schema, StructType, Transform}; +use crate::{Error, ErrorKind, Result}; + +/// Computes the unified partition type across all partition specs in the table. +/// +/// This is equivalent to Java's `Partitioning.partitionType(table)`. The result is a +/// StructType containing all partition fields ever used across all specs, enabling correct +/// representation of the `_partition` metadata column when partition evolution has occurred. +/// +/// Matches Java's behavior: +/// - Specs are sorted by spec_id in descending order (newer specs first), so newer field +/// names take precedence when deduplicating by field_id. +/// - Void and unknown transform fields are skipped. +/// - Fields are deduplicated by field_id — each unique field_id appears exactly once. +/// +/// # Arguments +/// * `partition_specs` - Iterator over all partition specs in the table +/// * `schema` - The current table schema (needed to determine result types of transforms) +pub fn compute_unified_partition_type<'a>( + partition_specs: impl Iterator<Item = &'a PartitionSpec>, + schema: &Schema, +) -> Result<StructType> { + let mut seen_field_ids = std::collections::HashSet::new(); + let mut struct_fields: Vec<NestedFieldRef> = Vec::new(); + + // Sort specs by spec_id descending (newer first) to match Java's behavior: + // newer field names take precedence when deduplicating by field_id. + let mut specs: Vec<&PartitionSpec> = partition_specs.collect(); + specs.sort_by_key(|s| std::cmp::Reverse(s.spec_id())); + + for spec in specs { + for field in spec.fields() { + if seen_field_ids.contains(&field.field_id) { + continue; + } + + // Skip void transforms (dropped partition columns) and unknown transforms + if matches!(field.transform, Transform::Void | Transform::Unknown) { Review Comment: fixed ########## crates/iceberg/src/arrow/record_batch_transformer.rs: ########## @@ -359,6 +430,20 @@ impl RecordBatchTransformer { let fields: Result<Vec<_>> = projected_iceberg_field_ids .iter() .map(|field_id| { + // Handle _partition struct column + if *field_id == RESERVED_FIELD_ID_PARTITION + && let Some(pc) = partition_column + { + let struct_type = DataType::Struct(pc.fields.clone()); + let nullable = pc.fields.is_empty(); + let arrow_field = Field::new("_partition", struct_type, nullable) Review Comment: fixed -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
