This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-3539-081a36dd311f4098a9a19847abedfa05f7af2e8f in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
commit 36aa0e435a13ed98e22af1851f81d16a3651c4d9 Author: Andy Grove <[email protected]> AuthorDate: Tue Oct 27 04:30:22 2026 +0000 fix(writer): skip NaN min/max when aggregating Parquet column bounds (#3539) * fix(writer): skip NaN min/max when aggregating Parquet column bounds Parquet 60 writes NaN min and max statistics for a column chunk whose values are all NaN, where earlier versions wrote none. MinMaxColAggregator copied them into the data file's lower and upper bounds, but the Iceberg spec does not allow NaN as a bound and iceberg-java records none. NaN is also unordered, so a stored NaN bound could not be replaced by a later chunk's value. * docs(writer): clarify why NaN bounds are skipped * fix(writer): correct NaN bound comment and cover negative NaN lower bound Co-authored-by: Copilot App <[email protected]> --------- Co-authored-by: Kevin Liu <[email protected]> Co-authored-by: Copilot App <[email protected]> --- .../src/writer/file_writer/parquet_writer.rs | 43 +++++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index a471aea5c..1470b310b 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -379,7 +379,10 @@ impl MinMaxColAggregator { )); }; - self.update_state_min(field_id, min_datum); + // Iceberg bounds must not be NaN. + if !min_datum.is_nan() { + self.update_state_min(field_id, min_datum); + } } if value.max_is_exact() { @@ -390,7 +393,9 @@ impl MinMaxColAggregator { )); }; - self.update_state_max(field_id, max_datum); + if !max_datum.is_nan() { + self.update_state_max(field_id, max_datum); + } } Ok(()) @@ -2605,6 +2610,40 @@ mod tests { assert_eq!(upper_bounds, HashMap::from([(0, Datum::int(i32::MAX))])); } + #[test] + fn test_min_max_aggregator_skips_nan_bounds() { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(0, "f", Type::Primitive(PrimitiveType::Float)).into(), + NestedField::required(1, "d", Type::Primitive(PrimitiveType::Double)).into(), + ]) + .build() + .expect("Failed to create schema"), + ); + let mut min_max_agg = MinMaxColAggregator::new(schema); + let float_stats = + |min, max| Statistics::Float(ValueStatistics::new(min, max, None, None, false)); + let double_stats = + |min, max| Statistics::Double(ValueStatistics::new(min, max, None, None, false)); + + min_max_agg + .update(0, &float_stats(Some(-f32::NAN), Some(f32::NAN))) + .unwrap(); + min_max_agg + .update(0, &float_stats(Some(1.5), Some(2.5))) + .unwrap(); + min_max_agg + .update(1, &double_stats(Some(f64::NAN), Some(f64::NAN))) + .unwrap(); + + let (lower_bounds, upper_bounds) = min_max_agg.produce(); + + assert_eq!(lower_bounds, HashMap::from([(0, Datum::float(1.5_f32))])); + assert_eq!(upper_bounds, HashMap::from([(0, Datum::float(2.5_f32))])); + } + fn cdc_test_schema() -> SchemaRef { Arc::new( Schema::builder()
