This is an automated email from the ASF dual-hosted git repository.
dheres pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 7ca7456d3a Handle nulls in approx_percentile_cont (#11721)
7ca7456d3a is described below
commit 7ca7456d3aa657b0b11c0d4eaae86576ae37a37e
Author: Daniël Heres <[email protected]>
AuthorDate: Wed Jul 31 08:14:34 2024 +0200
Handle nulls in approx_percentile_cont (#11721)
* Respect nulls in approx_percentile_cont
* Compile
* Remove check
* Adapt comment
---
datafusion/functions-aggregate/src/approx_percentile_cont.rs | 11 ++++++++---
datafusion/sqllogictest/test_files/aggregate.slt | 6 ++++++
2 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/datafusion/functions-aggregate/src/approx_percentile_cont.rs
b/datafusion/functions-aggregate/src/approx_percentile_cont.rs
index 16837dc807..844e48f0a4 100644
--- a/datafusion/functions-aggregate/src/approx_percentile_cont.rs
+++ b/datafusion/functions-aggregate/src/approx_percentile_cont.rs
@@ -19,7 +19,8 @@ use std::any::Any;
use std::fmt::{Debug, Formatter};
use std::sync::Arc;
-use arrow::array::RecordBatch;
+use arrow::array::{Array, RecordBatch};
+use arrow::compute::{filter, is_not_null};
use arrow::{
array::{
ArrayRef, Float32Array, Float64Array, Int16Array, Int32Array,
Int64Array,
@@ -393,8 +394,12 @@ impl Accumulator for ApproxPercentileAccumulator {
}
fn update_batch(&mut self, values: &[ArrayRef]) ->
datafusion_common::Result<()> {
- let values = &values[0];
- let sorted_values = &arrow::compute::sort(values, None)?;
+ // Remove any nulls before computing the percentile
+ let mut values = Arc::clone(&values[0]);
+ if values.nulls().is_some() {
+ values = filter(&values, &is_not_null(&values)?)?;
+ }
+ let sorted_values = &arrow::compute::sort(&values, None)?;
let sorted_values =
ApproxPercentileAccumulator::convert_to_float(sorted_values)?;
self.digest = self.digest.merge_sorted_f64(&sorted_values);
Ok(())
diff --git a/datafusion/sqllogictest/test_files/aggregate.slt
b/datafusion/sqllogictest/test_files/aggregate.slt
index 5cc66bb493..6ec1e0c526 100644
--- a/datafusion/sqllogictest/test_files/aggregate.slt
+++ b/datafusion/sqllogictest/test_files/aggregate.slt
@@ -1237,6 +1237,12 @@ SELECT (ABS(1 - CAST(approx_percentile_cont(c11, 0.9) AS
DOUBLE) / 0.834) < 0.05
----
true
+# percentile_cont_with_nulls
+query I
+SELECT APPROX_PERCENTILE_CONT(v, 0.5) FROM (VALUES (1), (2), (3), (NULL),
(NULL), (NULL)) as t (v);
+----
+2
+
# csv_query_cube_avg
query TIR
SELECT c1, c2, AVG(c3) FROM aggregate_test_100 GROUP BY CUBE (c1, c2) ORDER BY
c1, c2
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
