killme2008 commented on code in PR #27160:
URL: https://github.com/apache/datafusion/pull/27160#discussion_r4509565762
##########
datafusion/physical-expr/src/expressions/case.rs:
##########
@@ -385,6 +385,24 @@ fn filter_record_batch(
}
}
+fn take_case_rows<'a>(
+ batch: &'a RecordBatch,
+ indices: &dyn Array,
+) -> std::result::Result<Cow<'a, RecordBatch>, ArrowError> {
+ // These indices are an ordered subset of the original batch's rows.
+ if indices.len() == batch.num_rows() {
+ return Ok(Cow::Borrowed(batch));
Review Comment:
Added in
https://github.com/killme2008/datafusion/commit/afd0d365a79996ec5f55a97c2132e1760cfd0a38
##########
datafusion/physical-expr/benches/case_when.rs:
##########
@@ -87,6 +89,78 @@ fn criterion_benchmark(c: &mut Criterion) {
benchmark_lookup_table_case_when(c, 8192);
benchmark_divide_by_zero_protection(c, 8192);
+ benchmark_literal_when_selection(c, 8192);
+}
+
+fn benchmark_literal_when_selection(c: &mut Criterion, row_count: usize) {
+ let mut group = c.benchmark_group("literal_when_selection");
+ for (distribution, branches) in [
+ ("uniform", 1),
+ ("uniform", 4),
+ ("first_only", 4),
+ ("uniform", 16),
+ ("first_only", 16),
+ ("half_match_with_else", 1),
+ ] {
+ let first_only = distribution == "first_only";
+ let with_else = distribution == "half_match_with_else";
+ let partitions = if with_else { 2 } else { branches };
+ let batch = RecordBatch::try_from_iter(vec![
+ (
+ "partition",
+ Arc::new(Int32Array::from_iter_values((0..row_count).map(|i| {
+ if first_only {
+ 0
+ } else {
+ (i % partitions) as i32
+ }
+ }))) as ArrayRef,
+ ),
+ (
+ "left_key",
+ Arc::new(StringArray::from_iter_values(
+ (0..row_count).map(|i| format!("{i:032x}")),
+ )) as ArrayRef,
+ ),
+ (
+ "right_key",
+ Arc::new(StringArray::from_iter_values((0..row_count).map(|i| {
+ let value = if i % 3 == 0 { i } else { i + 1 };
+ format!("{value:032x}")
+ }))) as ArrayRef,
+ ),
+ ])
+ .unwrap();
+ let schema = batch.schema();
+ let then: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
+ col("left_key", &schema).unwrap(),
+ Operator::Eq,
+ col("right_key", &schema).unwrap(),
+ ));
+ let expr = case(
+ Some(col("partition", &schema).unwrap()),
+ (0..branches)
+ .map(|i| (lit(i as i32), Arc::clone(&then)))
+ .collect(),
+ with_else.then(|| Arc::clone(&then)),
+ )
+ .unwrap();
Review Comment:
Fixed in
https://github.com/killme2008/datafusion/commit/afd0d365a79996ec5f55a97c2132e1760cfd0a38
##########
datafusion/physical-expr/src/expressions/case.rs:
##########
@@ -1644,6 +1684,118 @@ mod tests {
}
}
+ #[test]
+ fn literal_when_preserves_row_indices_and_short_circuits() -> Result<()> {
+ let batch = RecordBatch::try_from_iter(vec![
+ (
+ "key",
+ Arc::new(Int32Array::from(vec![
+ Some(2),
+ None,
+ Some(1),
+ Some(2),
+ Some(3),
+ Some(4),
+ Some(0),
+ ])) as ArrayRef,
+ ),
+ (
+ "value",
+ Arc::new(Int32Array::from(vec![20, 99, 10, 21, 30, 40, 0])) as
ArrayRef,
+ ),
+ ])?;
+ let schema = batch.schema();
+ let invalid = binary(lit(1i32), Operator::Divide, lit(0i32), &schema)?;
+ for with_else in [false, true] {
+ let expr = case(
+ Some(col("key", &schema)?),
+ vec![
+ (lit(99i32), Arc::clone(&invalid)),
+ (lit(ScalarValue::Int32(None)), Arc::clone(&invalid)),
+ (lit(2i32), col("value", &schema)?),
+ (lit(1i32), col("value", &schema)?),
+ (lit(2i32), Arc::clone(&invalid)),
+ (lit(3i32), lit(300i32)),
+ ],
+ with_else.then(|| col("value", &schema)).transpose()?,
+ )?;
+ let expected = if with_else {
+ vec![
+ Some(20),
+ Some(99),
+ Some(10),
+ Some(21),
+ Some(300),
+ Some(40),
+ Some(0),
+ ]
+ } else {
+ vec![Some(20), None, Some(10), Some(21), Some(300), None, None]
+ };
Review Comment:
Done in
https://github.com/killme2008/datafusion/commit/afd0d365a79996ec5f55a97c2132e1760cfd0a38
##########
datafusion/physical-expr/src/expressions/case.rs:
##########
@@ -1644,6 +1684,118 @@ mod tests {
}
}
+ #[test]
+ fn literal_when_preserves_row_indices_and_short_circuits() -> Result<()> {
+ let batch = RecordBatch::try_from_iter(vec![
+ (
+ "key",
+ Arc::new(Int32Array::from(vec![
+ Some(2),
+ None,
+ Some(1),
+ Some(2),
+ Some(3),
+ Some(4),
+ Some(0),
+ ])) as ArrayRef,
+ ),
+ (
+ "value",
+ Arc::new(Int32Array::from(vec![20, 99, 10, 21, 30, 40, 0])) as
ArrayRef,
+ ),
+ ])?;
+ let schema = batch.schema();
+ let invalid = binary(lit(1i32), Operator::Divide, lit(0i32), &schema)?;
+ for with_else in [false, true] {
Review Comment:
Done in
https://github.com/killme2008/datafusion/commit/afd0d365a79996ec5f55a97c2132e1760cfd0a38
##########
datafusion/physical-expr/src/expressions/case.rs:
##########
@@ -1644,6 +1684,118 @@ mod tests {
}
}
+ #[test]
+ fn literal_when_preserves_row_indices_and_short_circuits() -> Result<()> {
+ let batch = RecordBatch::try_from_iter(vec![
+ (
+ "key",
+ Arc::new(Int32Array::from(vec![
+ Some(2),
+ None,
+ Some(1),
+ Some(2),
+ Some(3),
+ Some(4),
+ Some(0),
+ ])) as ArrayRef,
+ ),
+ (
+ "value",
+ Arc::new(Int32Array::from(vec![20, 99, 10, 21, 30, 40, 0])) as
ArrayRef,
+ ),
+ ])?;
+ let schema = batch.schema();
+ let invalid = binary(lit(1i32), Operator::Divide, lit(0i32), &schema)?;
+ for with_else in [false, true] {
+ let expr = case(
+ Some(col("key", &schema)?),
+ vec![
+ (lit(99i32), Arc::clone(&invalid)),
+ (lit(ScalarValue::Int32(None)), Arc::clone(&invalid)),
+ (lit(2i32), col("value", &schema)?),
+ (lit(1i32), col("value", &schema)?),
+ (lit(2i32), Arc::clone(&invalid)),
+ (lit(3i32), lit(300i32)),
+ ],
+ with_else.then(|| col("value", &schema)).transpose()?,
+ )?;
+ let expected = if with_else {
+ vec![
+ Some(20),
+ Some(99),
+ Some(10),
+ Some(21),
+ Some(300),
+ Some(40),
+ Some(0),
+ ]
+ } else {
+ vec![Some(20), None, Some(10), Some(21), Some(300), None, None]
+ };
+ // Four rows also exercise the branch that matches every remaining
row.
+ for count in [4, 7] {
+ let result = expr.evaluate(&batch.slice(0,
count))?.into_array(count)?;
+ assert_eq!(
+ as_int32_array(&result)?,
+ &Int32Array::from(expected[..count].to_vec())
+ );
+ }
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn taking_case_rows_preserves_zero_column_batch_length() -> Result<()> {
+ let batch = RecordBatch::try_new_with_options(
+ Arc::new(Schema::empty()),
+ vec![],
+ &RecordBatchOptions::new().with_row_count(Some(5)),
+ )?;
+ let selected = take_case_rows(&batch, &UInt32Array::from(vec![1, 3]))?;
+ assert_eq!(selected.num_rows(), 2);
+ Ok(())
+ }
+
+ #[test]
+ fn literal_when_preserves_nested_field_names() -> Result<()> {
+ let mut payload =
ListBuilder::new(Int32Builder::new()).with_field(Field::new(
+ "value",
+ DataType::Int32,
+ true,
+ ));
+ for value in [10, 20] {
+ payload.values().append_value(value);
+ payload.append(true);
+ }
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("key", DataType::Int32, false),
+ Field::new(
+ "payload",
+ DataType::List(Arc::new(Field::new("item", DataType::Int32,
true))),
+ false,
+ ),
+ ]));
+ let batch = RecordBatch::try_new_with_options(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(Int32Array::from(vec![1, 2])),
+ Arc::new(payload.finish()),
+ ],
+ &RecordBatchOptions::new().with_match_field_names(false),
+ )?;
Review Comment:
Removed the test in
https://github.com/apache/datafusion/pull/27160/commits/0e1e39466aef7a7e6b9e4502586ca87e69c69adc
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
