This is an automated email from the ASF dual-hosted git repository.
kazuyukitanimura pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 829656c0 fix: Optimize CheckOverflow (#852)
829656c0 is described below
commit 829656c0ea234861c6d4e6efd198b791a7bf4cc7
Author: KAZUYUKI TANIMURA <[email protected]>
AuthorDate: Mon Aug 19 18:50:42 2024 -0700
fix: Optimize CheckOverflow (#852)
## Which issue does this PR close?
## Rationale for this change
This PR improves CheckOverflow performance
## What changes are included in this PR?
This PR directly calls arrow validation methods
## How are these changes tested?
Existing tests
---
.../datafusion/expressions/checkoverflow.rs | 26 +++++-----------------
1 file changed, 5 insertions(+), 21 deletions(-)
diff --git a/native/core/src/execution/datafusion/expressions/checkoverflow.rs
b/native/core/src/execution/datafusion/expressions/checkoverflow.rs
index 044b366e..e4f54a1b 100644
--- a/native/core/src/execution/datafusion/expressions/checkoverflow.rs
+++ b/native/core/src/execution/datafusion/expressions/checkoverflow.rs
@@ -23,7 +23,7 @@ use std::{
};
use arrow::{
- array::{as_primitive_array, Array, ArrayRef, Decimal128Array,
PrimitiveArray},
+ array::{as_primitive_array, Array, ArrayRef, Decimal128Array},
datatypes::{Decimal128Type, DecimalType},
record_batch::RecordBatch,
};
@@ -111,30 +111,14 @@ impl PhysicalExpr for CheckOverflow {
let casted_array = if self.fail_on_error {
// Returning error if overflow
- let iter = decimal_array
- .iter()
- .map(|v| {
- v.map(|v| {
- Decimal128Type::validate_decimal_precision(v,
*precision).map(|_| v)
- })
- .map_or(Ok(None), |r| r.map(Some))
- })
- .collect::<Result<Vec<_>, _>>()?
- .into_iter();
- unsafe {
PrimitiveArray::<Decimal128Type>::from_trusted_len_iter(iter) }
+ decimal_array.validate_decimal_precision(*precision)?;
+ decimal_array
} else {
// Overflowing gets null value
- let iter = decimal_array.iter().map(|v| {
- v.and_then(|v| {
- Decimal128Type::validate_decimal_precision(v,
*precision)
- .map(|_| v)
- .ok()
- })
- });
- unsafe {
PrimitiveArray::<Decimal128Type>::from_trusted_len_iter(iter) }
+ &decimal_array.null_if_overflow_precision(*precision)
};
- let new_array = Decimal128Array::from(casted_array.to_data())
+ let new_array = Decimal128Array::from(casted_array.into_data())
.with_precision_and_scale(*precision, *scale)
.map(|a| Arc::new(a) as ArrayRef)?;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
