This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 4916e891c2 Boolean parquet get datapage stat (#11054)
4916e891c2 is described below
commit 4916e891c242220a5eaa02a602e497a03f5cf301
Author: Lorrens Pantelis <[email protected]>
AuthorDate: Sat Jun 22 00:30:29 2024 +0200
Boolean parquet get datapage stat (#11054)
* test and implement boolean data page statistics
* left out a collect & forgot to change the Check to Both
* Update datafusion/core/src/datasource/physical_plan/parquet/statistics.rs
---------
Co-authored-by: Andrew Lamb <[email protected]>
---
.../datasource/physical_plan/parquet/statistics.rs | 26 ++++++++++++++++++++++
datafusion/core/tests/parquet/arrow_statistics.rs | 2 +-
2 files changed, 27 insertions(+), 1 deletion(-)
diff --git a/datafusion/core/src/datasource/physical_plan/parquet/statistics.rs
b/datafusion/core/src/datasource/physical_plan/parquet/statistics.rs
index 3be060ce61..43483a0b66 100644
--- a/datafusion/core/src/datasource/physical_plan/parquet/statistics.rs
+++ b/datafusion/core/src/datasource/physical_plan/parquet/statistics.rs
@@ -549,6 +549,18 @@ macro_rules! make_data_page_stats_iterator {
};
}
+make_data_page_stats_iterator!(
+ MinBooleanDataPageStatsIterator,
+ |x: &PageIndex<bool>| { x.min },
+ Index::BOOLEAN,
+ bool
+);
+make_data_page_stats_iterator!(
+ MaxBooleanDataPageStatsIterator,
+ |x: &PageIndex<bool>| { x.max },
+ Index::BOOLEAN,
+ bool
+);
make_data_page_stats_iterator!(
MinInt32DataPageStatsIterator,
|x: &PageIndex<i32>| { x.min },
@@ -613,6 +625,15 @@ macro_rules! get_data_page_statistics {
($stat_type_prefix: ident, $data_type: ident, $iterator: ident) => {
paste! {
match $data_type {
+ Some(DataType::Boolean) => Ok(Arc::new(
+ BooleanArray::from_iter(
+ [<$stat_type_prefix
BooleanDataPageStatsIterator>]::new($iterator)
+ .flatten()
+ // BooleanArray::from_iter required a sized
iterator, so collect into Vec first
+ .collect::<Vec<_>>()
+ .into_iter()
+ )
+ )),
Some(DataType::UInt8) => Ok(Arc::new(
UInt8Array::from_iter(
[<$stat_type_prefix
Int32DataPageStatsIterator>]::new($iterator)
@@ -778,6 +799,11 @@ where
{
let iter = iterator.flat_map(|(len, index)| match index {
Index::NONE => vec![None; len],
+ Index::BOOLEAN(native_index) => native_index
+ .indexes
+ .iter()
+ .map(|x| x.null_count.map(|x| x as u64))
+ .collect::<Vec<_>>(),
Index::INT32(native_index) => native_index
.indexes
.iter()
diff --git a/datafusion/core/tests/parquet/arrow_statistics.rs
b/datafusion/core/tests/parquet/arrow_statistics.rs
index ddb39fce40..cd6985b311 100644
--- a/datafusion/core/tests/parquet/arrow_statistics.rs
+++ b/datafusion/core/tests/parquet/arrow_statistics.rs
@@ -1953,7 +1953,7 @@ async fn test_boolean() {
expected_null_counts: UInt64Array::from(vec![1, 0]),
expected_row_counts: Some(UInt64Array::from(vec![5, 5])),
column_name: "bool",
- check: Check::RowGroup,
+ check: Check::Both,
}
.run();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
