rgbuilds commented on code in PR #26441:
URL: https://github.com/apache/datafusion/pull/26441#discussion_r4507630916
##########
datafusion/catalog-listing/src/table.rs:
##########
@@ -1028,17 +1090,30 @@ impl ListingTable {
let (grouped, split_points) = file_group
.group_by_partition_values_with_split_points(file_group_count);
if grouped.len() >= threshold {
+ // Preserve the groups and split points derived from all
files so
+ // pruning cannot change the declared range partitioning.
+ let grouped = grouped
+ .into_iter()
+ .map(|group| {
+ self.prune_files_by_statistics(
+ pruning_predicate.as_deref(),
+ group,
+ )
+ })
+ .collect();
(grouped, true, split_points)
} else {
let all_files: Vec<_> =
grouped.into_iter().flat_map(|g|
g.into_inner()).collect();
- (
-
FileGroup::new(all_files).split_files(file_group_count),
- false,
- vec![],
- )
+ let file_group = self.prune_files_by_statistics(
+ pruning_predicate.as_deref(),
+ FileGroup::new(all_files),
+ );
Review Comment:
@jayzhan211 Added `preserve_partitions_below_threshold_still_prunes` with
two partition values and a threshold of 3. It verifies that pruning still
occurs before normal file splitting and produces `[[f2.parquet], [f3.parquet]]`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
