This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 4c8d7086dd Perf(parquet): RLE_dictionary reads; avoid round trip
allocation (#11474)
4c8d7086dd is described below
commit 4c8d7086dd4ec5aa97520a3a1a8cb7a866b31f34
Author: RIchard Baah <[email protected]>
AuthorDate: Sat Oct 17 08:18:16 2026 +0000
Perf(parquet): RLE_dictionary reads; avoid round trip allocation (#11474)
## part of https://github.com/apache/arrow-rs/pull/11423
Build the `DictionaryArray` directly from the typed keys/values instead
of going through `ArrayDataBuilder` + `make_array`, avoiding the
`ArrayData` round-trip and its associated validation/allocation on the
hot path in `DictionaryBuffer::into_array`.
Keys bounds are already validated, so construction uses
`DictionaryArray::new_unchecked`.
---
parquet/src/arrow/buffer/dictionary_buffer.rs | 66 +++++++++++++++++++--------
1 file changed, 48 insertions(+), 18 deletions(-)
diff --git a/parquet/src/arrow/buffer/dictionary_buffer.rs
b/parquet/src/arrow/buffer/dictionary_buffer.rs
index eb81058726..aef28b045b 100644
--- a/parquet/src/arrow/buffer/dictionary_buffer.rs
+++ b/parquet/src/arrow/buffer/dictionary_buffer.rs
@@ -19,14 +19,19 @@ use crate::arrow::buffer::offset_buffer::OffsetBuffer;
use crate::arrow::record_reader::buffer::ValuesBuffer;
use crate::errors::{ParquetError, Result};
use ahash::RandomState;
-use arrow_array::{Array, DictionaryArray, downcast_integer};
+use arrow_array::{Array, DictionaryArray, PrimitiveArray, downcast_integer};
use arrow_array::{
ArrayRef, FixedSizeBinaryArray, GenericBinaryArray, OffsetSizeTrait,
- builder::FixedSizeBinaryDictionaryBuilder, cast::AsArray, make_array,
- types::ArrowDictionaryKeyType,
+ builder::FixedSizeBinaryDictionaryBuilder,
+ cast::AsArray,
+ types::{
+ ArrowDictionaryKeyType, ArrowPrimitiveType, Int8Type, Int16Type,
Int32Type, Int64Type,
+ UInt8Type, UInt16Type, UInt32Type, UInt64Type,
+ },
+};
+use arrow_buffer::{
+ ArrowNativeType, BooleanBuffer, Buffer, MutableBuffer, NullBuffer,
ScalarBuffer,
};
-use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer};
-use arrow_data::ArrayDataBuilder;
use arrow_schema::DataType as ArrowType;
use hashbrown::HashMap as HbHashMap;
use hashbrown::hash_map::Entry;
@@ -159,7 +164,7 @@ impl<K: ArrowNativeType + Ord, V: OffsetSizeTrait>
DictionaryBuffer<K, V> {
}
}
- let ArrowType::Dictionary(_, value_type) = data_type else {
+ let ArrowType::Dictionary(key_type, value_type) = data_type
else {
unreachable!()
};
let values = if let ArrowType::FixedSizeBinary(size) =
**value_type {
@@ -173,18 +178,12 @@ impl<K: ArrowNativeType + Ord, V: OffsetSizeTrait>
DictionaryBuffer<K, V> {
values
};
- let builder = ArrayDataBuilder::new(data_type.clone())
- .len(keys.len())
- .add_buffer(Buffer::from_vec(keys))
- .add_child_data(values.into_data())
- .null_bit_buffer(null_buffer);
-
- let data = match cfg!(debug_assertions) {
- true => builder.build().unwrap(),
- false => unsafe { builder.build_unchecked() },
- };
+ let num_keys = keys.len();
+ let keys_buffer = Buffer::from_vec(keys);
+ let null_buffer =
+ null_buffer.map(|b| NullBuffer::new(BooleanBuffer::new(b,
0, num_keys)));
- Ok(make_array(data))
+ build_dictionary_array(key_type, keys_buffer, num_keys,
null_buffer, values)
}
Self::Values { values } => {
let (key_type, value_type) = match data_type {
@@ -245,6 +244,38 @@ impl<K: ArrowNativeType, V: OffsetSizeTrait> ValuesBuffer
for DictionaryBuffer<K
}
}
+/// Build a dictionary array from a raw keys buffer and an already-decoded
values
+/// `ArrayRef`, skipping the ArrayData round-trip.
+fn build_dictionary_array(
+ key_type: &ArrowType,
+ keys_buffer: Buffer,
+ num_keys: usize,
+ null_buffer: Option<NullBuffer>,
+ values: ArrayRef,
+) -> Result<ArrayRef> {
+ macro_rules! build {
+ ($kt:ty) => {{
+ let scalars =
+ ScalarBuffer::<<$kt as
ArrowPrimitiveType>::Native>::new(keys_buffer, 0, num_keys);
+ let keys = PrimitiveArray::<$kt>::new(scalars, null_buffer);
+ // SAFETY: key bounds already validated by caller
+ let dict = unsafe { DictionaryArray::<$kt>::new_unchecked(keys,
values) };
+ Ok(Arc::new(dict) as ArrayRef)
+ }};
+ }
+ match key_type {
+ ArrowType::Int8 => build!(Int8Type),
+ ArrowType::Int16 => build!(Int16Type),
+ ArrowType::Int32 => build!(Int32Type),
+ ArrowType::Int64 => build!(Int64Type),
+ ArrowType::UInt8 => build!(UInt8Type),
+ ArrowType::UInt16 => build!(UInt16Type),
+ ArrowType::UInt32 => build!(UInt32Type),
+ ArrowType::UInt64 => build!(UInt64Type),
+ _ => Err(general_err!("unsupported dictionary key type: {key_type}")),
+ }
+}
+
macro_rules! fixed_dict_helper {
($k:ty, $array:ident) => {
pack_fixed_values_impl::<$k>($array)
@@ -478,7 +509,6 @@ mod tests {
use super::*;
use arrow::compute::cast;
use arrow_array::StringArray;
- use arrow_array::types::*;
#[test]
fn test_dictionary_buffer() {
