geoffreyclaude commented on code in PR #23335:
URL: https://github.com/apache/datafusion/pull/23335#discussion_r3527987517
##########
datafusion/functions-nested/benches/array_has.rs:
##########
@@ -43,17 +45,224 @@ fn criterion_benchmark(c: &mut Criterion) {
for &size in &array_sizes {
bench_array_has(c, size);
+ bench_array_has_array(c, size);
bench_array_has_all(c, size);
bench_array_has_any(c, size);
}
// Specific benchmarks for string arrays (common use case)
bench_array_has_strings(c);
+ bench_array_has_array_strings(c);
bench_array_has_all_strings(c);
bench_array_has_any_strings(c);
// Benchmark for array_has_any with one scalar arg
bench_array_has_any_scalar(c);
+
+ // Array-needle fast-path profile: null patterns, list length, row height.
+ bench_array_has_array_null_patterns(c);
+ bench_array_has_array_by_size(c);
+ bench_array_has_array_by_rows(c);
+}
+
+/// Invoke `array_has` once with an array (column) needle -- exercises the
+/// `array_has_dispatch_for_array` fast path.
+fn run_array_needle_case(
+ group: &mut criterion::BenchmarkGroup<'_,
criterion::measurement::WallTime>,
+ id: String,
+ haystack: ArrayRef,
+ needle: ArrayRef,
+ rows: usize,
+) {
+ let config_options = Arc::new(ConfigOptions::default());
+ let return_field: Arc<Field> = Field::new("result", DataType::Boolean,
true).into();
+ let arg_fields: Vec<Arc<Field>> = vec![
+ Field::new("arr", haystack.data_type().clone(), false).into(),
+ Field::new("el", needle.data_type().clone(), false).into(),
+ ];
+ let args = vec![ColumnarValue::Array(haystack),
ColumnarValue::Array(needle)];
+ group.bench_function(id, |b| {
+ let udf = ArrayHas::new();
+ b.iter(|| {
+ black_box(
+ udf.invoke_with_args(ScalarFunctionArgs {
+ args: args.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: rows,
+ return_field: return_field.clone(),
+ config_options: config_options.clone(),
+ })
+ .unwrap(),
+ )
+ });
+ });
+}
+
+/// Build a `List` of `array_size` string elements per row (`{prefix}{i}`) with
+/// the given element type (`Utf8` / `LargeUtf8` / `Utf8View`) and null
density.
+/// The prefix controls element length: a short one stays inline in a
`Utf8View`
+/// (<= 12 bytes), a long one spills to the data buffer.
+fn string_list_array(
+ num_rows: usize,
+ array_size: usize,
+ null_density: f64,
+ prefix: &str,
+ element_type: &DataType,
+) -> ArrayRef {
+ let mut rng = StdRng::seed_from_u64(SEED);
+ let data = (0..num_rows * array_size).map(|_| {
+ if rng.random::<f64>() < null_density {
+ None
+ } else {
+ Some(format!("{prefix}{}", rng.random_range(0..array_size)))
+ }
+ });
+ let values: ArrayRef = match element_type {
+ DataType::Utf8 => Arc::new(data.collect::<StringArray>()),
+ DataType::LargeUtf8 => Arc::new(data.collect::<LargeStringArray>()),
+ DataType::Utf8View => Arc::new(data.collect::<StringViewArray>()),
+ other => panic!("unsupported string element type: {other}"),
+ };
+ let offsets = (0..=num_rows)
+ .map(|i| (i * array_size) as i32)
+ .collect::<Vec<i32>>();
+ Arc::new(
+ ListArray::try_new(
+ Arc::new(Field::new("item", element_type.clone(), true)),
+ OffsetBuffer::new(offsets.into()),
+ values,
+ None,
+ )
+ .unwrap(),
+ )
+}
+
+/// Build a string needle column (one value per row) of the given element type.
+fn string_value_array(
+ num_rows: usize,
+ range: usize,
+ prefix: &str,
+ element_type: &DataType,
+) -> ArrayRef {
+ let mut rng = StdRng::seed_from_u64(SEED + 2);
+ let data =
+ (0..num_rows).map(|_| Some(format!("{prefix}{}",
rng.random_range(0..range))));
+ match element_type {
+ DataType::Utf8 => Arc::new(data.collect::<StringArray>()),
+ DataType::LargeUtf8 => Arc::new(data.collect::<LargeStringArray>()),
+ DataType::Utf8View => Arc::new(data.collect::<StringViewArray>()),
+ other => panic!("unsupported string element type: {other}"),
+ }
+}
+
+/// Array needle, fixed list length (64), across null patterns. i64 covers no
+/// nulls / found / not-found / all-null / null-fill collision. Each string
+/// element type (`Utf8`, `LargeUtf8`, `Utf8View`) covers no nulls / 30% nulls
at
+/// both short (inline, <= 12 byte) and long (> 12 byte, shared-prefix) element
+/// lengths, plus all-null. `not_found` shifts the needle out of the value
range.
+fn bench_array_has_array_null_patterns(c: &mut Criterion) {
+ let (rows, size) = (10_000usize, 64usize);
+ let s = size as i64;
+ let mut group = c.benchmark_group("array_has_array_null_patterns");
+ run_array_needle_case(
Review Comment:
Can you just add:
```rust
run_array_needle_case(
&mut group,
"i64/no_nulls_not_found".to_string(),
create_int64_list_array(rows, size, 0.0),
create_int64_value_array(rows, s, s),
rows,
);
```
after this one so we cover the worst case "primitive, non-null, no match"?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
