This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-10636-103ebe7548db35c37e8ff35fcb02438bcd182fa7 in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
commit a1daa4a10b0c207963dce0b0d98be7640c2d32ca Author: cakeni <[email protected]> AuthorDate: Mon Oct 5 17:18:56 2026 +0000 [Arrow] Move parquet-variant-compute StructArrayAssembler to arrow-array (#10636) # Which issue does this PR close? - Closes #10618. # Rationale for this change `parquet-variant-compute` contains a local helper for assembling a `StructArray` from completed child arrays. The helper is generally useful Arrow functionality and should live in `arrow-array` instead of being duplicated in a Variant-specific crate. # What changes are included in this PR? - Add a public `StructArrayAssembler` for completed child arrays to `arrow-array`. - Support synthesized fields, caller-provided `FieldRef` values that retain metadata, and an optional top-level null buffer. - Replace the local `parquet-variant-compute` implementation with the shared assembler. - Add API documentation, an example, and unit coverage. # Are these changes tested? Yes. - `cargo test -p arrow-array struct_array_assembler` (16 tests passed) - `cargo test -p parquet-variant-compute` (347 unit tests and 13 doctests passed) # Are there any user-facing changes? Yes. `arrow-array` exposes a new assembler API for assembling `StructArray` values from completed children. Existing APIs are unchanged. ## AI assistance I investigated the existing builder design, determined the scope of the API extraction, and reviewed and refined the resulting API and implementation. OpenAI Codex assisted with code exploration, drafting, and test preparation. --------- Co-authored-by: cakeni <[email protected]> --- arrow-array/src/builder/mod.rs | 2 + arrow-array/src/builder/struct_array_assembler.rs | 163 ++++++++++++++++++++++ arrow-array/src/builder/struct_builder.rs | 3 + parquet-variant-compute/src/shred_variant.rs | 8 +- parquet-variant-compute/src/variant_array.rs | 81 +++-------- parquet-variant-compute/src/variant_get.rs | 23 +-- 6 files changed, 202 insertions(+), 78 deletions(-) diff --git a/arrow-array/src/builder/mod.rs b/arrow-array/src/builder/mod.rs index 813f028efb..189513b0a4 100644 --- a/arrow-array/src/builder/mod.rs +++ b/arrow-array/src/builder/mod.rs @@ -261,6 +261,8 @@ mod primitive_run_builder; pub use primitive_run_builder::*; mod struct_builder; pub use struct_builder::*; +mod struct_array_assembler; +pub use struct_array_assembler::*; mod generic_bytes_dictionary_builder; pub use generic_bytes_dictionary_builder::*; mod generic_byte_run_builder; diff --git a/arrow-array/src/builder/struct_array_assembler.rs b/arrow-array/src/builder/struct_array_assembler.rs new file mode 100644 index 0000000000..f8ebde093e --- /dev/null +++ b/arrow-array/src/builder/struct_array_assembler.rs @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::{ArrayRef, StructArray}; +use arrow_buffer::NullBuffer; +use arrow_schema::{ArrowError, Field, FieldRef, Fields}; +use std::sync::Arc; + +/// Assembles a [`StructArray`] from completed child arrays. +/// +/// Unlike [`StructBuilder`](super::StructBuilder), which incrementally builds +/// child arrays, this assembler combines arrays that have already been built. +/// +/// # Example +/// +/// ``` +/// # use std::sync::Arc; +/// # use arrow_array::builder::StructArrayAssembler; +/// # use arrow_array::{Array, ArrayRef, Int32Array, StringArray}; +/// +/// let names = Arc::new(StringArray::from(vec!["one", "two"])) as ArrayRef; +/// let values = Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef; +/// // Create struct array with `{name: ..., value: ...}` rows +/// let array = StructArrayAssembler::new() +/// .with_field("name", names, false) +/// .with_field("value", values, false) +/// .build() +/// .unwrap(); +/// +/// assert_eq!(array.len(), 2); +/// ``` +#[derive(Debug, Default, Clone)] +pub struct StructArrayAssembler { + fields: Vec<FieldRef>, + arrays: Vec<ArrayRef>, + nulls: Option<NullBuffer>, +} + +impl StructArrayAssembler { + /// Creates a new empty [`StructArrayAssembler`]. + pub fn new() -> Self { + Self::default() + } + + /// Adds an array with a field constructed from its data type. + pub fn with_field( + mut self, + field_name: impl Into<String>, + array: ArrayRef, + nullable: bool, + ) -> Self { + self.fields.push(Arc::new(Field::new( + field_name, + array.data_type().clone(), + nullable, + ))); + self.arrays.push(array); + self + } + + /// Adds an array using a caller-supplied field. + /// + /// This preserves field metadata that would be lost if the field were + /// constructed from the array's data type alone. + pub fn with_field_ref(mut self, field: FieldRef, array: ArrayRef) -> Self { + self.fields.push(field); + self.arrays.push(array); + self + } + + /// Sets the top-level null buffer for the struct array. + pub fn with_nulls(mut self, nulls: NullBuffer) -> Self { + self.nulls = Some(nulls); + self + } + + /// Builds the [`StructArray`]. + /// + /// # Errors + /// + /// Returns an error if the fields, child arrays, or null buffer have + /// incompatible data types or lengths. + pub fn build(self) -> Result<StructArray, ArrowError> { + StructArray::try_new(Fields::from(self.fields), self.arrays, self.nulls) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Array, Int32Array, StringArray}; + use arrow_schema::DataType; + use std::collections::HashMap; + + #[test] + fn build_from_completed_arrays() { + let names = Arc::new(StringArray::from(vec!["one", "two"])) as ArrayRef; + let values = Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef; + let metadata = HashMap::from([("key".to_string(), "value".to_string())]); + let value_field = + Arc::new(Field::new("value", DataType::Int32, false).with_metadata(metadata.clone())); + let nulls = NullBuffer::from(vec![true, false]); + + let array = StructArrayAssembler::new() + .with_field("name", names, false) + .with_field_ref(value_field, values) + .with_nulls(nulls.clone()) + .build() + .unwrap(); + + assert_eq!(array.len(), 2); + assert_eq!(array.column_names(), &["name", "value"]); + assert_eq!(array.fields()[1].metadata(), &metadata); + assert_eq!(array.nulls(), Some(&nulls)); + } + + #[test] + fn build_errors_on_mismatched_child_lengths() { + let names = Arc::new(StringArray::from(vec!["one", "two"])) as ArrayRef; + let values = Arc::new(Int32Array::from(vec![1])) as ArrayRef; + + let err = StructArrayAssembler::new() + .with_field("name", names, false) + .with_field("value", values, false) + .build() + .unwrap_err(); + + assert_eq!( + err.to_string(), + "Invalid argument error: Incorrect array length for StructArray field \"value\", expected 2 got 1" + ); + } + + #[test] + fn build_errors_on_mismatched_field_data_type() { + let value_field = Arc::new(Field::new("value", DataType::Int64, false)); + let values = Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef; + + let err = StructArrayAssembler::new() + .with_field_ref(value_field, values) + .build() + .unwrap_err(); + + assert_eq!( + err.to_string(), + "Invalid argument error: Incorrect datatype for StructArray field \"value\", expected Int64 got Int32" + ); + } +} diff --git a/arrow-array/src/builder/struct_builder.rs b/arrow-array/src/builder/struct_builder.rs index e8af63cb2d..3362396803 100644 --- a/arrow-array/src/builder/struct_builder.rs +++ b/arrow-array/src/builder/struct_builder.rs @@ -26,6 +26,9 @@ use std::sync::Arc; /// Note that callers should make sure that methods of all the child field builders are /// properly called to maintain the consistency of the data structure. /// +/// For assembling a [`StructArray`] from child arrays that are already built, see +/// [`StructArrayAssembler`]. +/// /// /// Handling arrays with complex layouts, such as `List<Struct<List<Struct>>>`, in Rust can be challenging due to its strong typing system. /// To construct a collection builder ([`ListBuilder`], [`LargeListBuilder`], or [`MapBuilder`]) using [`make_builder`], multiple calls are required. This complexity arises from the recursive approach utilized by [`StructBuilder::from_fields`]. diff --git a/parquet-variant-compute/src/shred_variant.rs b/parquet-variant-compute/src/shred_variant.rs index 2d78bc9014..c48df074d2 100644 --- a/parquet-variant-compute/src/shred_variant.rs +++ b/parquet-variant-compute/src/shred_variant.rs @@ -18,13 +18,13 @@ //! Module for shredding VariantArray with a given schema. use crate::type_conversion::VariantCastMode; -use crate::variant_array::{ShreddedVariantFieldArray, StructArrayBuilder}; +use crate::variant_array::ShreddedVariantFieldArray; use crate::variant_to_arrow::{ ArrayVariantToArrowRowBuilder, PrimitiveVariantToArrowRowBuilder, make_primitive_variant_to_arrow_row_builder, }; use crate::{VariantArray, VariantValueArrayBuilder}; -use arrow::array::{ArrayRef, BinaryViewArray, NullBufferBuilder}; +use arrow::array::{ArrayRef, BinaryViewArray, NullBufferBuilder, StructArrayAssembler}; use arrow::buffer::NullBuffer; use arrow::compute::CastOptions; use arrow::datatypes::{DataType, Field, FieldRef, Fields, TimeUnit}; @@ -460,7 +460,7 @@ impl<'a> VariantToShreddedObjectVariantRowBuilder<'a> { } fn finish(mut self) -> Result<(BinaryViewArray, ArrayRef, Option<NullBuffer>)> { - let mut builder = StructArrayBuilder::new(); + let mut builder = StructArrayAssembler::new(); for (field_name, typed_value_builder) in self.typed_value_builders { let (value, typed_value, nulls) = typed_value_builder.finish()?; let array = @@ -472,7 +472,7 @@ impl<'a> VariantToShreddedObjectVariantRowBuilder<'a> { } Ok(( self.value_builder.build()?, - Arc::new(builder.build()), + Arc::new(builder.build()?), self.nulls.finish(), )) } diff --git a/parquet-variant-compute/src/variant_array.rs b/parquet-variant-compute/src/variant_array.rs index cdf3436f31..8327421f0b 100644 --- a/parquet-variant-compute/src/variant_array.rs +++ b/parquet-variant-compute/src/variant_array.rs @@ -23,8 +23,8 @@ use crate::type_conversion::{ primitive_conversion_single_value, }; use arrow::array::{ - Array, ArrayRef, AsArray, StructArray, downcast_dictionary_array, downcast_run_array, - new_null_array, + Array, ArrayRef, AsArray, StructArray, StructArrayAssembler, downcast_dictionary_array, + downcast_run_array, new_null_array, }; use arrow::buffer::NullBuffer; use arrow::compute::cast; @@ -35,7 +35,7 @@ use arrow::datatypes::{ }; use arrow::error::Result; use arrow_schema::extension::{ExtensionType, Uuid as UuidExtension}; -use arrow_schema::{ArrowError, DataType, Field, FieldRef, Fields, TimeUnit}; +use arrow_schema::{ArrowError, DataType, Field, FieldRef, TimeUnit}; use chrono::{DateTime, NaiveTime}; use parquet_variant::{ Uuid, Variant, VariantDecimal4, VariantDecimal8, VariantDecimal16, VariantDecimalType as _, @@ -419,7 +419,7 @@ impl VariantArray { nulls: Option<NullBuffer>, value_nullable: bool, ) -> Self { - let mut builder = StructArrayBuilder::new() + let mut builder = StructArrayAssembler::new() .with_field("metadata", metadata.clone(), false) .with_field("value", value.clone(), value_nullable); if let Some(typed_value) = typed_value.clone() { @@ -430,7 +430,7 @@ impl VariantArray { } Self { - inner: builder.build(), + inner: builder.build().unwrap(), metadata, shredding_state: ShreddingState::new(value, typed_value), } @@ -837,7 +837,7 @@ impl ShreddedVariantFieldArray { typed_value: Option<ArrayRef>, nulls: Option<NullBuffer>, ) -> Self { - let mut builder = StructArrayBuilder::new().with_field("value", value.clone(), true); + let mut builder = StructArrayAssembler::new().with_field("value", value.clone(), true); if let Some(typed_value) = typed_value.clone() { builder = builder.with_field_ref(typed_value_field(&typed_value), typed_value); } @@ -846,7 +846,7 @@ impl ShreddedVariantFieldArray { } Self { - inner: builder.build(), + inner: builder.build().unwrap(), shredding_state: ShreddingState::new(value, typed_value), } } @@ -1017,55 +1017,6 @@ fn typed_value_field(array: &ArrayRef) -> FieldRef { Arc::new(field) } -/// Builds struct arrays from component fields -/// -/// TODO: move to arrow crate -#[derive(Debug, Default, Clone)] -pub(crate) struct StructArrayBuilder { - fields: Vec<FieldRef>, - arrays: Vec<ArrayRef>, - nulls: Option<NullBuffer>, -} - -impl StructArrayBuilder { - pub fn new() -> Self { - Default::default() - } - - /// Add an array to this struct array as a field with the specified name. - pub fn with_field(mut self, field_name: &str, array: ArrayRef, nullable: bool) -> Self { - let field = Field::new(field_name, array.data_type().clone(), nullable); - self.fields.push(Arc::new(field)); - self.arrays.push(array); - self - } - - /// Add an array to this struct array using a caller-supplied [`FieldRef`]. - /// - /// Use this when the field carries metadata (e.g. an extension type) that - /// would be lost if the field were synthesized from the array's data type alone. - pub fn with_field_ref(mut self, field: FieldRef, array: ArrayRef) -> Self { - self.fields.push(field); - self.arrays.push(array); - self - } - - /// Set the null buffer for this struct array. - pub fn with_nulls(mut self, nulls: NullBuffer) -> Self { - self.nulls = Some(nulls); - self - } - - pub fn build(self) -> StructArray { - let Self { - fields, - arrays, - nulls, - } = self; - StructArray::new(Fields::from(fields), arrays, nulls) - } -} - /// returns the non-null element at index as a Variant fn typed_value_to_variant(typed_value: &ArrayRef, index: usize) -> Result<Variant<'_, '_>> { let data_type = typed_value.data_type(); @@ -1457,10 +1408,11 @@ mod test { let typed_value = Arc::new(Int64Array::from(vec![Some(1), None, Some(3)])) as ArrayRef; let metadata = BinaryViewArray::from_iter_values(std::iter::repeat_n(EMPTY_VARIANT_METADATA_BYTES, 3)); - let struct_array = StructArrayBuilder::new() + let struct_array = StructArrayAssembler::new() .with_field("metadata", Arc::new(metadata), false) .with_field("typed_value", typed_value, true) - .build(); + .build() + .unwrap(); assert!(struct_array.column_by_name("value").is_none()); let variant_array = VariantArray::try_new(&struct_array).unwrap(); @@ -1596,11 +1548,12 @@ mod test { typed_value.len(), )); let value = new_null_array(&DataType::BinaryView, typed_value.len()); - StructArrayBuilder::new() + StructArrayAssembler::new() .with_field("metadata", Arc::new(metadata), false) .with_field("value", value, true) .with_field("typed_value", typed_value, true) .build() + .unwrap() } #[test] @@ -1622,12 +1575,14 @@ mod test { // A shredded object { id: { typed_value: FixedSizeBinary(16) } } whose inner UUID leaf // carries no extension type; canonicalization must reach it recursively. let leaf = FixedSizeBinaryArray::try_from_iter(std::iter::repeat_n([0u8; 16], 1)).unwrap(); - let inner = StructArrayBuilder::new() + let inner = StructArrayAssembler::new() .with_field("typed_value", Arc::new(leaf), true) - .build(); - let object = StructArrayBuilder::new() + .build() + .unwrap(); + let object = StructArrayAssembler::new() .with_field("id", Arc::new(inner), false) - .build(); + .build() + .unwrap(); let input = make_variant_struct_with_typed_value(Arc::new(object)); // typed_value (struct) -> id (struct) -> typed_value (the FixedSizeBinary(16) UUID leaf). diff --git a/parquet-variant-compute/src/variant_get.rs b/parquet-variant-compute/src/variant_get.rs index 97cc3b00d9..9ec38f0aa1 100644 --- a/parquet-variant-compute/src/variant_get.rs +++ b/parquet-variant-compute/src/variant_get.rs @@ -517,9 +517,7 @@ mod test { use std::sync::Arc; use super::{GetOptions, requested_field_is_shredded, variant_get}; - use crate::variant_array::{ - ShreddedVariantFieldArray, StructArrayBuilder, all_null_value_column, - }; + use crate::variant_array::{ShreddedVariantFieldArray, all_null_value_column}; use crate::{ ShreddedSchemaBuilder, VariantArray, VariantArrayBuilder, cast_to_variant, json_to_variant, shred_variant, @@ -531,8 +529,8 @@ mod test { Int64Array, Int64Builder, LargeBinaryArray, LargeListArray, LargeListViewArray, LargeStringArray, ListArray, ListBuilder, ListViewArray, MapBuilder, NullArray, NullBuilder, StringArray, StringBuilder, StringViewArray, StructArray, - Time32MillisecondArray, Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray, - UnionArray, + StructArrayAssembler, Time32MillisecondArray, Time32SecondArray, Time64MicrosecondArray, + Time64NanosecondArray, UnionArray, }; use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow::compute::{CastOptions, cast}; @@ -3797,10 +3795,11 @@ mod test { false, // row 2: outer field NULL false, // row 3: top-level NULL ]); - let outer_typed_value = StructArrayBuilder::new() + let outer_typed_value = StructArrayAssembler::new() .with_field("inner", ArrayRef::from(inner), false) .with_nulls(outer_typed_value_nulls) - .build(); + .build() + .unwrap(); let outer = ShreddedVariantFieldArray::perfectly_shredded(Arc::new(outer_typed_value) as ArrayRef); @@ -3811,10 +3810,11 @@ mod test { false, // row 2: outer field NULL false, // row 3: top-level NULL ]); - let typed_value = StructArrayBuilder::new() + let typed_value = StructArrayAssembler::new() .with_field("outer", ArrayRef::from(outer), false) .with_nulls(typed_value_nulls) - .build(); + .build() + .unwrap(); // Build final VariantArray with top-level nulls let metadata_array = @@ -3884,9 +3884,10 @@ mod test { ) as ArrayRef); // Create main typed_value struct (only contains shredded fields) - let typed_value_struct = StructArrayBuilder::new() + let typed_value_struct = StructArrayAssembler::new() .with_field("x", ArrayRef::from(x_field_shredded), false) - .build(); + .build() + .unwrap(); // Build VariantArray with both value and typed_value (PartiallyShredded) // Top-level null is encoded in the main StructArray's null mask
