This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-11861-14c2fef335c4396552d60d69502950f2d0424c70 in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
commit adcec44f4c093f24ef75d424691639f000a34577 Author: Yi Yao <[email protected]> AuthorDate: Wed Oct 28 15:12:18 2026 +0000 Reject logical nulls in non-nullable RecordBatch columns (#11832) (#11861) # Which issue does this PR close? - Closes #11832 # Rationale for this change `RecordBatch::try_new` previously validated non-nullable fields using `c.null_count() > 0`, which only checked the top-level validity buffer. For arrays whose nulls reside in child arrays or dictionary values (such as `UnionArray`, `RunArray`, or `DictionaryArray`), `c.null_count()` is `0` even when logical nulls exist, causing `RecordBatch::try_new` to accept logical nulls in non-nullable columns. Other array constructors like `StructArray::try_new`, `ListArray::try_new`, and `FixedSizeListArray::try_new` already check logical nulls and reject these cases. # What changes are included in this PR? - Check `c.logical_null_count() > 0` in `RecordBatch::try_new` for non-nullable fields. - Update `RecordBatch::try_from_iter` to determine column nullability via `array.logical_null_count() > 0`. - Add unit tests verifying that `UnionArray`, `RunArray`, and `DictionaryArray` containing nulls are rejected in non-nullable `RecordBatch` columns, and that `try_from_iter` marks such columns nullable. # Are these changes tested? Yes, new unit tests were added in `arrow-array/src/record_batch.rs` covering `UnionArray`, `RunArray`, and `DictionaryArray` logical null verification in `RecordBatch::try_new` and `RecordBatch::try_from_iter`. # Are there any significant user-facing changes? Non-nullable fields in `RecordBatch::try_new` will now correctly return an error if passed columns containing logical nulls (aligning with `StructArray`, `ListArray`, and `FixedSizeListArray`). --------- Co-authored-by: Yi Yao <[email protected]> --- arrow-array/src/record_batch.rs | 83 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/arrow-array/src/record_batch.rs b/arrow-array/src/record_batch.rs index 0c0b495533..ee06fdeb32 100644 --- a/arrow-array/src/record_batch.rs +++ b/arrow-array/src/record_batch.rs @@ -339,7 +339,7 @@ impl RecordBatch { })?; for (c, f) in columns.iter().zip(&schema.fields) { - if !f.is_nullable() && c.null_count() > 0 { + if !f.is_nullable() && c.logical_null_count() > 0 { return Err(ArrowError::InvalidArgumentError(format!( "Column '{}' is declared as non-nullable but contains null values", f.name() @@ -733,7 +733,7 @@ impl RecordBatch { // https://github.com/rust-lang/rust/issues/50133 is no longer an // issue let iter = value.into_iter().map(|(field_name, array)| { - let nullable = array.null_count() > 0; + let nullable = array.logical_null_count() > 0; (field_name, array, nullable) }); @@ -1740,6 +1740,85 @@ mod tests { format!("{}", maybe_batch.err().unwrap()) ); } + + #[test] + fn test_validate_logical_nulls_union() { + let fields = + arrow_schema::UnionFields::try_new([0], [Field::new("u", DataType::Int32, true)]) + .unwrap(); + let child = Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef; + let union_array = Arc::new( + crate::UnionArray::try_new(fields, vec![0, 0].into(), None, vec![child]).unwrap(), + ); + let schema = Arc::new(Schema::new(vec![Field::new( + "a", + union_array.data_type().clone(), + false, + )])); + let maybe_batch = RecordBatch::try_new(schema, vec![union_array]); + assert_eq!( + "Invalid argument error: Column 'a' is declared as non-nullable but contains null values", + format!("{}", maybe_batch.err().unwrap()) + ); + } + + #[test] + fn test_validate_logical_nulls_run_array() { + let values = Int32Array::from(vec![None]); + let run_array = Arc::new( + crate::RunArray::<crate::types::Int32Type>::try_new( + &Int32Array::from(vec![2]), + &values, + ) + .unwrap(), + ); + let schema = Arc::new(Schema::new(vec![Field::new( + "a", + run_array.data_type().clone(), + false, + )])); + let maybe_batch = RecordBatch::try_new(schema, vec![run_array]); + assert_eq!( + "Invalid argument error: Column 'a' is declared as non-nullable but contains null values", + format!("{}", maybe_batch.err().unwrap()) + ); + } + + #[test] + fn test_validate_logical_nulls_dictionary() { + let values = Arc::new(Int32Array::from(vec![None])); + let dict_array = Arc::new( + crate::DictionaryArray::<crate::types::Int8Type>::try_new( + Int8Array::from(vec![0, 0]), + values, + ) + .unwrap(), + ); + let schema = Arc::new(Schema::new(vec![Field::new( + "a", + dict_array.data_type().clone(), + false, + )])); + let maybe_batch = RecordBatch::try_new(schema, vec![dict_array]); + assert_eq!( + "Invalid argument error: Column 'a' is declared as non-nullable but contains null values", + format!("{}", maybe_batch.err().unwrap()) + ); + } + + #[test] + fn test_try_from_iter_logical_nulls() { + let values = Arc::new(Int32Array::from(vec![None])); + let dict_array = Arc::new( + crate::DictionaryArray::<crate::types::Int8Type>::try_new( + Int8Array::from(vec![0, 0]), + values, + ) + .unwrap(), + ); + let batch = RecordBatch::try_from_iter(vec![("d", dict_array as ArrayRef)]).unwrap(); + assert!(batch.schema().field(0).is_nullable()); + } #[test] fn test_record_batch_options() { let options = RecordBatchOptions::new()
