This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 58d23c5c05 feat: support uint data page extraction (#11018)
58d23c5c05 is described below
commit 58d23c5c050f43aa7b867d4f0be7298d8d6cad83
Author: Trent Hauck <[email protected]>
AuthorDate: Thu Jun 20 05:59:56 2024 -0700
feat: support uint data page extraction (#11018)
---
.../datasource/physical_plan/parquet/statistics.rs | 42 ++++++++++++++++++++++
datafusion/core/tests/parquet/arrow_statistics.rs | 8 ++---
2 files changed, 46 insertions(+), 4 deletions(-)
diff --git a/datafusion/core/src/datasource/physical_plan/parquet/statistics.rs
b/datafusion/core/src/datasource/physical_plan/parquet/statistics.rs
index 2ca47de990..3be060ce61 100644
--- a/datafusion/core/src/datasource/physical_plan/parquet/statistics.rs
+++ b/datafusion/core/src/datasource/physical_plan/parquet/statistics.rs
@@ -613,6 +613,48 @@ macro_rules! get_data_page_statistics {
($stat_type_prefix: ident, $data_type: ident, $iterator: ident) => {
paste! {
match $data_type {
+ Some(DataType::UInt8) => Ok(Arc::new(
+ UInt8Array::from_iter(
+ [<$stat_type_prefix
Int32DataPageStatsIterator>]::new($iterator)
+ .map(|x| {
+ x.into_iter().filter_map(|x| {
+ x.and_then(|x| u8::try_from(x).ok())
+ })
+ })
+ .flatten()
+ )
+ )),
+ Some(DataType::UInt16) => Ok(Arc::new(
+ UInt16Array::from_iter(
+ [<$stat_type_prefix
Int32DataPageStatsIterator>]::new($iterator)
+ .map(|x| {
+ x.into_iter().filter_map(|x| {
+ x.and_then(|x| u16::try_from(x).ok())
+ })
+ })
+ .flatten()
+ )
+ )),
+ Some(DataType::UInt32) => Ok(Arc::new(
+ UInt32Array::from_iter(
+ [<$stat_type_prefix
Int32DataPageStatsIterator>]::new($iterator)
+ .map(|x| {
+ x.into_iter().filter_map(|x| {
+ x.and_then(|x| u32::try_from(x).ok())
+ })
+ })
+ .flatten()
+ ))),
+ Some(DataType::UInt64) => Ok(Arc::new(
+ UInt64Array::from_iter(
+ [<$stat_type_prefix
Int64DataPageStatsIterator>]::new($iterator)
+ .map(|x| {
+ x.into_iter().filter_map(|x| {
+ x.and_then(|x| u64::try_from(x).ok())
+ })
+ })
+ .flatten()
+ ))),
Some(DataType::Int8) => Ok(Arc::new(
Int8Array::from_iter(
[<$stat_type_prefix
Int32DataPageStatsIterator>]::new($iterator)
diff --git a/datafusion/core/tests/parquet/arrow_statistics.rs
b/datafusion/core/tests/parquet/arrow_statistics.rs
index bdae9f4786..e48c11fcba 100644
--- a/datafusion/core/tests/parquet/arrow_statistics.rs
+++ b/datafusion/core/tests/parquet/arrow_statistics.rs
@@ -1352,7 +1352,7 @@ async fn test_uint() {
expected_null_counts: UInt64Array::from(vec![0, 0, 0, 0, 0]),
expected_row_counts: Some(UInt64Array::from(vec![4, 4, 4, 4, 4])),
column_name: "u8",
- check: Check::RowGroup,
+ check: Check::Both,
}
.run();
@@ -1363,7 +1363,7 @@ async fn test_uint() {
expected_null_counts: UInt64Array::from(vec![0, 0, 0, 0, 0]),
expected_row_counts: Some(UInt64Array::from(vec![4, 4, 4, 4, 4])),
column_name: "u16",
- check: Check::RowGroup,
+ check: Check::Both,
}
.run();
@@ -1374,7 +1374,7 @@ async fn test_uint() {
expected_null_counts: UInt64Array::from(vec![0, 0, 0, 0, 0]),
expected_row_counts: Some(UInt64Array::from(vec![4, 4, 4, 4, 4])),
column_name: "u32",
- check: Check::RowGroup,
+ check: Check::Both,
}
.run();
@@ -1385,7 +1385,7 @@ async fn test_uint() {
expected_null_counts: UInt64Array::from(vec![0, 0, 0, 0, 0]),
expected_row_counts: Some(UInt64Array::from(vec![4, 4, 4, 4, 4])),
column_name: "u64",
- check: Check::RowGroup,
+ check: Check::Both,
}
.run();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
