This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-11474-9827aaad30258bf348a0d4f2b792505d6d24a88f in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
commit 4c8d7086dd4ec5aa97520a3a1a8cb7a866b31f34 Author: RIchard Baah <[email protected]> AuthorDate: Sat Oct 17 08:18:16 2026 +0000 Perf(parquet): RLE_dictionary reads; avoid round trip allocation (#11474) ## part of https://github.com/apache/arrow-rs/pull/11423 Build the `DictionaryArray` directly from the typed keys/values instead of going through `ArrayDataBuilder` + `make_array`, avoiding the `ArrayData` round-trip and its associated validation/allocation on the hot path in `DictionaryBuffer::into_array`. Keys bounds are already validated, so construction uses `DictionaryArray::new_unchecked`. --- parquet/src/arrow/buffer/dictionary_buffer.rs | 66 +++++++++++++++++++-------- 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/parquet/src/arrow/buffer/dictionary_buffer.rs b/parquet/src/arrow/buffer/dictionary_buffer.rs index eb81058726..aef28b045b 100644 --- a/parquet/src/arrow/buffer/dictionary_buffer.rs +++ b/parquet/src/arrow/buffer/dictionary_buffer.rs @@ -19,14 +19,19 @@ use crate::arrow::buffer::offset_buffer::OffsetBuffer; use crate::arrow::record_reader::buffer::ValuesBuffer; use crate::errors::{ParquetError, Result}; use ahash::RandomState; -use arrow_array::{Array, DictionaryArray, downcast_integer}; +use arrow_array::{Array, DictionaryArray, PrimitiveArray, downcast_integer}; use arrow_array::{ ArrayRef, FixedSizeBinaryArray, GenericBinaryArray, OffsetSizeTrait, - builder::FixedSizeBinaryDictionaryBuilder, cast::AsArray, make_array, - types::ArrowDictionaryKeyType, + builder::FixedSizeBinaryDictionaryBuilder, + cast::AsArray, + types::{ + ArrowDictionaryKeyType, ArrowPrimitiveType, Int8Type, Int16Type, Int32Type, Int64Type, + UInt8Type, UInt16Type, UInt32Type, UInt64Type, + }, +}; +use arrow_buffer::{ + ArrowNativeType, BooleanBuffer, Buffer, MutableBuffer, NullBuffer, ScalarBuffer, }; -use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer}; -use arrow_data::ArrayDataBuilder; use arrow_schema::DataType as ArrowType; use hashbrown::HashMap as HbHashMap; use hashbrown::hash_map::Entry; @@ -159,7 +164,7 @@ impl<K: ArrowNativeType + Ord, V: OffsetSizeTrait> DictionaryBuffer<K, V> { } } - let ArrowType::Dictionary(_, value_type) = data_type else { + let ArrowType::Dictionary(key_type, value_type) = data_type else { unreachable!() }; let values = if let ArrowType::FixedSizeBinary(size) = **value_type { @@ -173,18 +178,12 @@ impl<K: ArrowNativeType + Ord, V: OffsetSizeTrait> DictionaryBuffer<K, V> { values }; - let builder = ArrayDataBuilder::new(data_type.clone()) - .len(keys.len()) - .add_buffer(Buffer::from_vec(keys)) - .add_child_data(values.into_data()) - .null_bit_buffer(null_buffer); - - let data = match cfg!(debug_assertions) { - true => builder.build().unwrap(), - false => unsafe { builder.build_unchecked() }, - }; + let num_keys = keys.len(); + let keys_buffer = Buffer::from_vec(keys); + let null_buffer = + null_buffer.map(|b| NullBuffer::new(BooleanBuffer::new(b, 0, num_keys))); - Ok(make_array(data)) + build_dictionary_array(key_type, keys_buffer, num_keys, null_buffer, values) } Self::Values { values } => { let (key_type, value_type) = match data_type { @@ -245,6 +244,38 @@ impl<K: ArrowNativeType, V: OffsetSizeTrait> ValuesBuffer for DictionaryBuffer<K } } +/// Build a dictionary array from a raw keys buffer and an already-decoded values +/// `ArrayRef`, skipping the ArrayData round-trip. +fn build_dictionary_array( + key_type: &ArrowType, + keys_buffer: Buffer, + num_keys: usize, + null_buffer: Option<NullBuffer>, + values: ArrayRef, +) -> Result<ArrayRef> { + macro_rules! build { + ($kt:ty) => {{ + let scalars = + ScalarBuffer::<<$kt as ArrowPrimitiveType>::Native>::new(keys_buffer, 0, num_keys); + let keys = PrimitiveArray::<$kt>::new(scalars, null_buffer); + // SAFETY: key bounds already validated by caller + let dict = unsafe { DictionaryArray::<$kt>::new_unchecked(keys, values) }; + Ok(Arc::new(dict) as ArrayRef) + }}; + } + match key_type { + ArrowType::Int8 => build!(Int8Type), + ArrowType::Int16 => build!(Int16Type), + ArrowType::Int32 => build!(Int32Type), + ArrowType::Int64 => build!(Int64Type), + ArrowType::UInt8 => build!(UInt8Type), + ArrowType::UInt16 => build!(UInt16Type), + ArrowType::UInt32 => build!(UInt32Type), + ArrowType::UInt64 => build!(UInt64Type), + _ => Err(general_err!("unsupported dictionary key type: {key_type}")), + } +} + macro_rules! fixed_dict_helper { ($k:ty, $array:ident) => { pack_fixed_values_impl::<$k>($array) @@ -478,7 +509,6 @@ mod tests { use super::*; use arrow::compute::cast; use arrow_array::StringArray; - use arrow_array::types::*; #[test] fn test_dictionary_buffer() {
