This is an automated email from the ASF dual-hosted git repository.
jayzhan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 2653c63a45 Minor: add Arc for statistics in FileGroup (#15564)
2653c63a45 is described below
commit 2653c63a45eb1ec9b65c85abf562a376031a566a
Author: xudong.w <[email protected]>
AuthorDate: Fri Apr 4 08:17:57 2025 +0800
Minor: add Arc for statistics in FileGroup (#15564)
---
datafusion/datasource/src/file_groups.rs | 7 ++++---
datafusion/datasource/src/statistics.rs | 2 +-
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/datafusion/datasource/src/file_groups.rs
b/datafusion/datasource/src/file_groups.rs
index 75c4160f14..a1f966c22f 100644
--- a/datafusion/datasource/src/file_groups.rs
+++ b/datafusion/datasource/src/file_groups.rs
@@ -25,6 +25,7 @@ use std::collections::BinaryHeap;
use std::iter::repeat_with;
use std::mem;
use std::ops::{Index, IndexMut};
+use std::sync::Arc;
/// Repartition input files into `target_partitions` partitions, if total file
size exceed
/// `repartition_file_min_size`
@@ -368,7 +369,7 @@ pub struct FileGroup {
/// The files in this group
files: Vec<PartitionedFile>,
/// Optional statistics for the data across all files in the group
- statistics: Option<Statistics>,
+ statistics: Option<Arc<Statistics>>,
}
impl FileGroup {
@@ -386,7 +387,7 @@ impl FileGroup {
}
/// Set the statistics for this group
- pub fn with_statistics(mut self, statistics: Statistics) -> Self {
+ pub fn with_statistics(mut self, statistics: Arc<Statistics>) -> Self {
self.statistics = Some(statistics);
self
}
@@ -420,7 +421,7 @@ impl FileGroup {
/// Get the statistics for this group
pub fn statistics(&self) -> Option<&Statistics> {
- self.statistics.as_ref()
+ self.statistics.as_deref()
}
/// Partition the list of files into `n` groups
diff --git a/datafusion/datasource/src/statistics.rs
b/datafusion/datasource/src/statistics.rs
index 7e875513f0..040bf754dd 100644
--- a/datafusion/datasource/src/statistics.rs
+++ b/datafusion/datasource/src/statistics.rs
@@ -494,7 +494,7 @@ pub fn compute_file_group_statistics(
file.statistics.as_ref().map(|stats| stats.as_ref())
});
- Ok(file_group.with_statistics(statistics))
+ Ok(file_group.with_statistics(Arc::new(statistics)))
}
/// Computes statistics for all files across multiple file groups.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
