This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 378b9eecd4 chore: Improve performance of Parquet statistics conversion
(#10932)
378b9eecd4 is described below
commit 378b9eecd4a77386a59953209f75fc5c192d7af4
Author: Alex Huang <[email protected]>
AuthorDate: Mon Jun 17 17:43:20 2024 +0800
chore: Improve performance of Parquet statistics conversion (#10932)
---
.../datasource/physical_plan/parquet/statistics.rs | 32 +++-------------------
1 file changed, 4 insertions(+), 28 deletions(-)
diff --git a/datafusion/core/src/datasource/physical_plan/parquet/statistics.rs
b/datafusion/core/src/datasource/physical_plan/parquet/statistics.rs
index a2e0d8fa66..327a516f1a 100644
--- a/datafusion/core/src/datasource/physical_plan/parquet/statistics.rs
+++ b/datafusion/core/src/datasource/physical_plan/parquet/statistics.rs
@@ -303,24 +303,12 @@ macro_rules! get_statistics {
))),
DataType::Int8 => Ok(Arc::new(Int8Array::from_iter(
[<$stat_type_prefix
Int32StatsIterator>]::new($iterator).map(|x| {
- x.and_then(|x| {
- if let Ok(v) = i8::try_from(*x) {
- Some(v)
- } else {
- None
- }
- })
+ x.and_then(|x| i8::try_from(*x).ok())
}),
))),
DataType::Int16 => Ok(Arc::new(Int16Array::from_iter(
[<$stat_type_prefix
Int32StatsIterator>]::new($iterator).map(|x| {
- x.and_then(|x| {
- if let Ok(v) = i16::try_from(*x) {
- Some(v)
- } else {
- None
- }
- })
+ x.and_then(|x| i16::try_from(*x).ok())
}),
))),
DataType::Int32 => Ok(Arc::new(Int32Array::from_iter(
@@ -331,24 +319,12 @@ macro_rules! get_statistics {
))),
DataType::UInt8 => Ok(Arc::new(UInt8Array::from_iter(
[<$stat_type_prefix
Int32StatsIterator>]::new($iterator).map(|x| {
- x.and_then(|x| {
- if let Ok(v) = u8::try_from(*x) {
- Some(v)
- } else {
- None
- }
- })
+ x.and_then(|x| u8::try_from(*x).ok())
}),
))),
DataType::UInt16 => Ok(Arc::new(UInt16Array::from_iter(
[<$stat_type_prefix
Int32StatsIterator>]::new($iterator).map(|x| {
- x.and_then(|x| {
- if let Ok(v) = u16::try_from(*x) {
- Some(v)
- } else {
- None
- }
- })
+ x.and_then(|x| u16::try_from(*x).ok())
}),
))),
DataType::UInt32 => Ok(Arc::new(UInt32Array::from_iter(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
