This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch 57_maintenance
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/57_maintenance by this push:
new 8926973eb8 [57_maintenance] fix: implement
`ArrayData::validate_values` for union types (#11814)
8926973eb8 is described below
commit 8926973eb80a8a5e2c1dadea7ae6928fd99c63db
Author: Jeffrey Vo <[email protected]>
AuthorDate: Fri Oct 23 00:26:41 2026 +0900
[57_maintenance] fix: implement `ArrayData::validate_values` for union
types (#11814)
- Backport https://github.com/apache/arrow-rs/pull/11534
- For inclusion in https://github.com/apache/arrow-rs/issues/11666
Cherry-pick of
https://github.com/apache/arrow-rs/commit/23ac45ed817d3200e69113473746942f0615ed47
Co-authored-by: Andrew Lamb <[email protected]>
---
arrow-data/src/data.rs | 121 ++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 114 insertions(+), 7 deletions(-)
diff --git a/arrow-data/src/data.rs b/arrow-data/src/data.rs
index 9d98f75643..b6989a6a0d 100644
--- a/arrow-data/src/data.rs
+++ b/arrow-data/src/data.rs
@@ -1377,12 +1377,47 @@ impl ArrayData {
let child = &self.child_data[0];
self.validate_offsets_full::<i64>(child.len)
}
- DataType::Union(_, _) => {
- // Validate Union Array as part of implementing new Union
semantics
- // See comments in `ArrayData::validate()`
- // https://github.com/apache/arrow-rs/issues/85
- //
- // TODO file follow on ticket for full union validation
+ DataType::Union(fields, mode) => {
+ let mut child_lens: Vec<Option<usize>> = vec![None; 128];
+ for (index, (type_id, _)) in fields.iter().enumerate() {
+ let child = &self.child_data[index];
+ child_lens[type_id as usize] = Some(child.len());
+ }
+
+ // Type id values must match one of the fields.
+ let type_ids = self.typed_buffer::<i8>(0, self.len)?;
+ for (index, &type_id) in type_ids.iter().enumerate() {
+ if type_id < 0 {
+ return Err(ArrowError::InvalidArgumentError(format!(
+ "Type id values cannot be negative, got {type_id}
at index {index}"
+ )));
+ }
+ let type_id = type_id as usize;
+ if child_lens[type_id].is_none() {
+ return Err(ArrowError::InvalidArgumentError(format!(
+ "Type id values must match one of the field type
ids, got {type_id} at index {index}"
+ )));
+ }
+ }
+
+ // Check the value offsets are in bounds.
+ if mode == &UnionMode::Dense {
+ let offsets = self.typed_buffer::<i32>(1, self.len)?;
+
+ if type_ids
+ .iter()
+ .zip(offsets.iter())
+ .any(|(&type_id, &offset)| {
+ offset < 0 || offset as usize >=
child_lens[type_id as usize].unwrap()
+ })
+ {
+ return Err(ArrowError::InvalidArgumentError(
+ "Offsets must be non-negative and within the
length of the child array"
+ .to_string(),
+ ));
+ }
+ }
+
Ok(())
}
DataType::Dictionary(key_type, _value_type) => {
@@ -2194,7 +2229,7 @@ mod tests {
use crate::transform::MutableArrayData;
use super::*;
- use arrow_schema::{Field, Fields};
+ use arrow_schema::{Field, Fields, UnionFields};
// See arrow/tests/array_data_validation.rs for test of array validation
@@ -2658,6 +2693,78 @@ mod tests {
}
}
+ #[test]
+ fn test_union_validate_type_ids() {
+ let dt = DataType::Union(
+ UnionFields::try_new(
+ vec![5],
+ vec![Arc::new(Field::new("a", DataType::Int32, true))],
+ )
+ .unwrap(),
+ UnionMode::Sparse,
+ );
+
+ let child = ArrayData::builder(DataType::Int32)
+ .len(2)
+ .add_buffer(make_i32_buffer(2))
+ .build()
+ .unwrap();
+
+ let err = ArrayDataBuilder::new(dt)
+ .len(2)
+ .offset(0)
+ .buffers(vec![vec![5_i8, 2].into()])
+ .child_data(vec![child])
+ .build()
+ .unwrap_err();
+ assert_eq!(
+ err.to_string(),
+ "Invalid argument error: Type id values must match one of the
field type ids, got 2 at index 1"
+ );
+ }
+
+ #[test]
+ fn test_union_validate_dense_offsets() {
+ let dt = DataType::Union(
+ UnionFields::try_new(
+ vec![5],
+ vec![Arc::new(Field::new("a", DataType::Int32, true))],
+ )
+ .unwrap(),
+ UnionMode::Dense,
+ );
+
+ let child = ArrayData::builder(DataType::Int32)
+ .len(2)
+ .add_buffer(make_i32_buffer(2))
+ .build()
+ .unwrap();
+
+ let err = ArrayDataBuilder::new(dt.clone())
+ .len(2)
+ // 2nd offset '10' is out of bounds
+ .buffers(vec![vec![5_i8, 5].into(), vec![0_i32, 10].into()])
+ .child_data(vec![child.clone()])
+ .build()
+ .unwrap_err();
+ assert_eq!(
+ err.to_string(),
+ "Invalid argument error: Offsets must be non-negative and within
the length of the child array"
+ );
+
+ let err = ArrayDataBuilder::new(dt)
+ .len(2)
+ // 2nd offset is negative
+ .buffers(vec![vec![5_i8, 5].into(), vec![0_i32, -1].into()])
+ .child_data(vec![child])
+ .build()
+ .unwrap_err();
+ assert_eq!(
+ err.to_string(),
+ "Invalid argument error: Offsets must be non-negative and within
the length of the child array"
+ );
+ }
+
/// Returns a `RunEndEncoded<Int32, Float32>` data type and run ends /
values
/// children whose runs end at `[1, 3]`, i.e. a logical length of 3.
fn run_array_test_children() -> (DataType, Vec<ArrayData>) {
