Rich-T-kid commented on code in PR #27008:
URL: https://github.com/apache/datafusion/pull/27008#discussion_r4450733028
##########
datafusion/common/benches/with_hashes.rs:
##########
@@ -645,10 +670,33 @@ fn mixed_columns_benchmark(c: &mut Criterion) {
});
}
+/// Int32 dictionaries with 50% and 97% null keys. 0% and 3% are covered by
+/// the existing `dictionary_utf8_int32` "no nulls" and "nulls" cases in
+/// `criterion_benchmark`
+fn dictionary_benchmark(c: &mut Criterion) {
+ let pool = StringPool::new(100, 64);
+ let dict = pool.typed_dictionary_array::<Int32Type>(BATCH_SIZE);
+ let name = "dictionary_utf8_int32";
+ for null_percent in [50, 97] {
+ let array = with_null_keys(&dict, null_percent as f32 / 100.0);
+ c.bench_function(&format!("{name}: single, {null_percent}% null
keys"), |b| {
+ do_hash_test(b, std::slice::from_ref(&array));
+ });
+ c.bench_function(
+ &format!("{name}: multiple, {null_percent}% null keys"),
+ |b| {
+ let arrays = vec![array.clone(), array.clone(), array.clone()];
+ do_hash_test(b, &arrays);
+ },
+ );
+ }
+}
Review Comment:
I think we should add this to existing benchmarks
##########
datafusion/common/benches/with_hashes.rs:
##########
@@ -290,9 +309,15 @@ impl StringPool {
&self,
array_length: usize,
) -> ArrayRef {
- Arc::new(DictionaryArray::<T>::from_iter(
- self.iter_strings(array_length),
- ))
+ Arc::new(self.typed_dictionary_array::<T>(array_length))
+ }
+
+ /// Like [`Self::dictionary_array`], but returns the concrete
`DictionaryArray`
+ fn typed_dictionary_array<T: ArrowDictionaryKeyType>(
+ &self,
+ array_length: usize,
+ ) -> DictionaryArray<T> {
+ DictionaryArray::<T>::from_iter(self.iter_strings(array_length))
Review Comment:
why do we need to return the concrete `DictionaryArray` here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
