rluvaton commented on code in PR #27160:
URL: https://github.com/apache/datafusion/pull/27160#discussion_r4502962278
##########
datafusion/physical-expr/src/expressions/case.rs:
##########
@@ -385,6 +385,24 @@ fn filter_record_batch(
}
}
+fn take_case_rows<'a>(
+ batch: &'a RecordBatch,
+ indices: &dyn Array,
+) -> std::result::Result<Cow<'a, RecordBatch>, ArrowError> {
+ // These indices are an ordered subset of the original batch's rows.
+ if indices.len() == batch.num_rows() {
+ return Ok(Cow::Borrowed(batch));
Review Comment:
Maybe add debug assert that it equal 0..batch.len()
##########
datafusion/physical-expr/benches/case_when.rs:
##########
@@ -87,6 +89,78 @@ fn criterion_benchmark(c: &mut Criterion) {
benchmark_lookup_table_case_when(c, 8192);
benchmark_divide_by_zero_protection(c, 8192);
+ benchmark_literal_when_selection(c, 8192);
+}
+
+fn benchmark_literal_when_selection(c: &mut Criterion, row_count: usize) {
+ let mut group = c.benchmark_group("literal_when_selection");
+ for (distribution, branches) in [
+ ("uniform", 1),
+ ("uniform", 4),
+ ("first_only", 4),
+ ("uniform", 16),
+ ("first_only", 16),
+ ("half_match_with_else", 1),
+ ] {
+ let first_only = distribution == "first_only";
+ let with_else = distribution == "half_match_with_else";
+ let partitions = if with_else { 2 } else { branches };
+ let batch = RecordBatch::try_from_iter(vec![
+ (
+ "partition",
+ Arc::new(Int32Array::from_iter_values((0..row_count).map(|i| {
+ if first_only {
+ 0
+ } else {
+ (i % partitions) as i32
+ }
+ }))) as ArrayRef,
+ ),
+ (
+ "left_key",
+ Arc::new(StringArray::from_iter_values(
+ (0..row_count).map(|i| format!("{i:032x}")),
+ )) as ArrayRef,
+ ),
+ (
+ "right_key",
+ Arc::new(StringArray::from_iter_values((0..row_count).map(|i| {
+ let value = if i % 3 == 0 { i } else { i + 1 };
+ format!("{value:032x}")
+ }))) as ArrayRef,
+ ),
+ ])
+ .unwrap();
+ let schema = batch.schema();
+ let then: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
+ col("left_key", &schema).unwrap(),
+ Operator::Eq,
+ col("right_key", &schema).unwrap(),
+ ));
+ let expr = case(
+ Some(col("partition", &schema).unwrap()),
+ (0..branches)
+ .map(|i| (lit(i as i32), Arc::clone(&then)))
+ .collect(),
+ with_else.then(|| Arc::clone(&then)),
+ )
+ .unwrap();
Review Comment:
Can you please avoid the same then in or branches and else, later
optimization could make this benchmark useless
##########
datafusion/physical-expr/src/expressions/case.rs:
##########
@@ -1644,6 +1684,118 @@ mod tests {
}
}
+ #[test]
+ fn literal_when_preserves_row_indices_and_short_circuits() -> Result<()> {
+ let batch = RecordBatch::try_from_iter(vec![
+ (
+ "key",
+ Arc::new(Int32Array::from(vec![
+ Some(2),
+ None,
+ Some(1),
+ Some(2),
+ Some(3),
+ Some(4),
+ Some(0),
+ ])) as ArrayRef,
+ ),
+ (
+ "value",
+ Arc::new(Int32Array::from(vec![20, 99, 10, 21, 30, 40, 0])) as
ArrayRef,
+ ),
+ ])?;
+ let schema = batch.schema();
+ let invalid = binary(lit(1i32), Operator::Divide, lit(0i32), &schema)?;
+ for with_else in [false, true] {
+ let expr = case(
+ Some(col("key", &schema)?),
+ vec![
+ (lit(99i32), Arc::clone(&invalid)),
+ (lit(ScalarValue::Int32(None)), Arc::clone(&invalid)),
+ (lit(2i32), col("value", &schema)?),
+ (lit(1i32), col("value", &schema)?),
+ (lit(2i32), Arc::clone(&invalid)),
+ (lit(3i32), lit(300i32)),
+ ],
+ with_else.then(|| col("value", &schema)).transpose()?,
+ )?;
+ let expected = if with_else {
+ vec![
+ Some(20),
+ Some(99),
+ Some(10),
+ Some(21),
+ Some(300),
+ Some(40),
+ Some(0),
+ ]
+ } else {
+ vec![Some(20), None, Some(10), Some(21), Some(300), None, None]
+ };
Review Comment:
Just create 2 tests
##########
datafusion/physical-expr/src/expressions/case.rs:
##########
@@ -385,6 +385,24 @@ fn filter_record_batch(
}
}
+fn take_case_rows<'a>(
+ batch: &'a RecordBatch,
+ indices: &dyn Array,
+) -> std::result::Result<Cow<'a, RecordBatch>, ArrowError> {
+ // These indices are an ordered subset of the original batch's rows.
+ if indices.len() == batch.num_rows() {
+ return Ok(Cow::Borrowed(batch));
+ }
+ let columns = arrow::compute::take_arrays(batch.columns(), indices, None)?;
+ Ok(Cow::Owned(RecordBatch::try_new_with_options(
+ batch.schema(),
+ columns,
+ &RecordBatchOptions::new()
+ .with_match_field_names(false)
Review Comment:
Why?
##########
datafusion/physical-expr/src/expressions/case.rs:
##########
@@ -1644,6 +1684,118 @@ mod tests {
}
}
+ #[test]
+ fn literal_when_preserves_row_indices_and_short_circuits() -> Result<()> {
+ let batch = RecordBatch::try_from_iter(vec![
+ (
+ "key",
+ Arc::new(Int32Array::from(vec![
+ Some(2),
+ None,
+ Some(1),
+ Some(2),
+ Some(3),
+ Some(4),
+ Some(0),
+ ])) as ArrayRef,
+ ),
+ (
+ "value",
+ Arc::new(Int32Array::from(vec![20, 99, 10, 21, 30, 40, 0])) as
ArrayRef,
+ ),
+ ])?;
+ let schema = batch.schema();
+ let invalid = binary(lit(1i32), Operator::Divide, lit(0i32), &schema)?;
+ for with_else in [false, true] {
Review Comment:
Please create 2 tests
##########
datafusion/physical-expr/src/expressions/case.rs:
##########
@@ -385,6 +385,24 @@ fn filter_record_batch(
}
}
+fn take_case_rows<'a>(
+ batch: &'a RecordBatch,
+ indices: &dyn Array,
+) -> std::result::Result<Cow<'a, RecordBatch>, ArrowError> {
+ // These indices are an ordered subset of the original batch's rows.
+ if indices.len() == batch.num_rows() {
+ return Ok(Cow::Borrowed(batch));
+ }
+ let columns = arrow::compute::take_arrays(batch.columns(), indices, None)?;
+ Ok(Cow::Owned(RecordBatch::try_new_with_options(
+ batch.schema(),
+ columns,
+ &RecordBatchOptions::new()
+ .with_match_field_names(false)
+ .with_row_count(Some(indices.len())),
+ )?))
Review Comment:
If the indices are ordered and not duplicated indices why not use filter?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
