This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-11909-a404d6f9a07e82f4fc3dd52a1e188df2588a292a in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
commit 16310a9f7b94463e583dc7b70c812333e2cba75d Author: Y Ethan Guo <[email protected]> AuthorDate: Mon Oct 26 20:11:18 2026 +0000 bench(parquet): add FIXED_LEN_BYTE_ARRAY(7) decimal reader benchmarks with negative values (#11909) # Which issue does this PR close? - Part of #11908 # Rationale for this change The `FIXED_LEN_BYTE_ARRAY` decimal benchmarks only cover 16-byte, non-negative values, while parquet-mr based writers use the minimal width for the precision (7 bytes for `DECIMAL(15,2)`). This adds that case ahead of the change for #11908 so it can run on the benchmark runner. # What changes are included in this PR? Two groups read a `FIXED_LEN_BYTE_ARRAY(7)` `DECIMAL(15,2)` column with negative values, as `Decimal128` and, through a type hint, as `Decimal64`. The generator now writes the low `type_length` bytes of each value, which leaves the existing 16-byte data unchanged. # Are these changes tested? They are benchmarks. # Are there any significant user-facing changes? No. AI usage: written with an AI coding assistant under my direction; I reviewed the code and ran the benchmarks. Co-authored-by: Andrew Lamb <[email protected]> --- parquet/benches/arrow_reader.rs | 68 +++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/parquet/benches/arrow_reader.rs b/parquet/benches/arrow_reader.rs index 3b23628009..d7a5a6686d 100644 --- a/parquet/benches/arrow_reader.rs +++ b/parquet/benches/arrow_reader.rs @@ -113,6 +113,8 @@ fn build_test_schema() -> SchemaDescPtr { } REQUIRED BOOLEAN mandatory_bool_leaf; OPTIONAL BOOLEAN optional_bool_leaf; + REQUIRED FIXED_LEN_BYTE_ARRAY (7) mandatory_decimal5_leaf (DECIMAL(15,2)); + OPTIONAL FIXED_LEN_BYTE_ARRAY (7) optional_decimal5_leaf (DECIMAL(15,2)); } "; parse_message_type(message_type) @@ -242,10 +244,9 @@ where big_int.to_signed_bytes_be() } Type::FIXED_LEN_BYTE_ARRAY => { - assert_eq!(column_desc.type_length(), 16); - // fixed length byte array use the fixed size - // the size is 16 - value.to_be_bytes().to_vec() + // the low `type_length` bytes of the big-endian value + let width = column_desc.type_length() as usize; + value.to_be_bytes()[16 - width..].to_vec() } _ => unimplemented!(), }; @@ -1072,13 +1073,14 @@ fn create_f16_by_bytes_reader( fn create_decimal_by_bytes_reader( page_iterator: impl PageIterator + 'static, column_desc: ColumnDescPtr, + arrow_type: Option<DataType>, ) -> Box<dyn ArrayReader> { let physical_type = column_desc.physical_type(); match physical_type { Type::BYTE_ARRAY => make_byte_array_reader( Box::new(page_iterator), column_desc, - None, + arrow_type, DEFAULT_BATCH_SIZE, None, ) @@ -1086,7 +1088,7 @@ fn create_decimal_by_bytes_reader( Type::FIXED_LEN_BYTE_ARRAY => make_fixed_len_byte_array_reader( Box::new(page_iterator), column_desc, - None, + arrow_type, DEFAULT_BATCH_SIZE, None, ) @@ -1194,6 +1196,7 @@ fn bench_byte_decimal<T>( encoding: Encoding, min: i128, max: i128, + arrow_type: Option<DataType>, ) where T: parquet::data_type::DataType, T::T: From<Vec<u8>>, @@ -1213,8 +1216,11 @@ fn bench_byte_decimal<T>( encoding.to_string().to_lowercase() + " encoded, mandatory, no NULLs", |b| { b.iter(|| { - let array_reader = - create_decimal_by_bytes_reader(data.clone(), mandatory_column_desc.clone()); + let array_reader = create_decimal_by_bytes_reader( + data.clone(), + mandatory_column_desc.clone(), + arrow_type.clone(), + ); count = bench_array_reader(array_reader); }); assert_eq!(count, EXPECTED_VALUE_COUNT); @@ -1232,8 +1238,11 @@ fn bench_byte_decimal<T>( encoding.to_string().to_lowercase() + " encoded, optional, no NULLs", |b| { b.iter(|| { - let array_reader = - create_decimal_by_bytes_reader(data.clone(), optional_column_desc.clone()); + let array_reader = create_decimal_by_bytes_reader( + data.clone(), + optional_column_desc.clone(), + arrow_type.clone(), + ); count = bench_array_reader(array_reader); }); assert_eq!(count, EXPECTED_VALUE_COUNT); @@ -1252,8 +1261,11 @@ fn bench_byte_decimal<T>( encoding.to_string().to_lowercase() + " encoded, optional, half NULLs", |b| { b.iter(|| { - let array_reader = - create_decimal_by_bytes_reader(data.clone(), optional_column_desc.clone()); + let array_reader = create_decimal_by_bytes_reader( + data.clone(), + optional_column_desc.clone(), + arrow_type.clone(), + ); count = bench_array_reader(array_reader); }); assert_eq!(count, EXPECTED_VALUE_COUNT); @@ -1961,6 +1973,7 @@ fn decimal_benches(c: &mut Criterion) { // precision is 16: the max is 9999999999999999 9999999999999000, 9999999999999999, + None, ); group.finish(); @@ -1976,6 +1989,7 @@ fn decimal_benches(c: &mut Criterion) { // precision is 16: the max is 9999999999999999 9999999999999000, 9999999999999999, + None, ); group.finish(); @@ -1990,6 +2004,36 @@ fn decimal_benches(c: &mut Criterion) { // precision is 16: the max is 9999999999999999 9999999999999000, 9999999999999999, + None, + ); + group.finish(); + + // parquet FIXED_LEN_BYTE_ARRAY(7), logical type decimal(15,2): the minimal + // width, as Spark's legacy format writes it + let mut group = c.benchmark_group("arrow_array_reader/FIXED_LEN_BYTE_ARRAY(7)/Decimal128Array"); + let mandatory_decimal5_leaf_desc = schema.column(45); + let optional_decimal5_leaf_desc = schema.column(46); + bench_byte_decimal::<FixedLenByteArrayType>( + &mut group, + &mandatory_decimal5_leaf_desc, + &optional_decimal5_leaf_desc, + Encoding::PLAIN, + // precision is 15, with negative values to exercise sign extension + -999999999999999, + 999999999999999, + None, + ); + group.finish(); + + let mut group = c.benchmark_group("arrow_array_reader/FIXED_LEN_BYTE_ARRAY(7)/Decimal64Array"); + bench_byte_decimal::<FixedLenByteArrayType>( + &mut group, + &mandatory_decimal5_leaf_desc, + &optional_decimal5_leaf_desc, + Encoding::PLAIN, + -999999999999999, + 999999999999999, + Some(DataType::Decimal64(15, 2)), ); group.finish(); }
