viirya commented on code in PR #3284:
URL: https://github.com/apache/arrow-rs/pull/3284#discussion_r1043080239
##########
parquet/src/arrow/arrow_writer/mod.rs:
##########
@@ -1312,6 +1314,70 @@ mod tests {
values_optional::<A, I>(iter);
}
+ fn check_bloom_filter<T: AsBytes>(
+ files: Vec<File>,
+ file_column: String,
+ positive_values: Vec<T>,
+ negative_values: Vec<T>,
+ ) {
+ files.into_iter().for_each(|file| {
+ let file_reader = SerializedFileReader::new_with_options(
+ file,
+ ReadOptionsBuilder::new()
+ .with_reader_properties(
+ ReaderProperties::builder()
+ .set_read_bloom_filter(true)
+ .build(),
+ )
+ .build(),
+ )
+ .expect("Unable to open file as Parquet");
+ let metadata = file_reader.metadata();
+ for (ri, row_group) in metadata.row_groups().iter().enumerate() {
+ if let Some((column_index, _)) = row_group
+ .columns()
+ .iter()
+ .enumerate()
+ .find(|(_, column)| column.column_path().string() ==
file_column)
+ {
+ let row_group_reader = file_reader
+ .get_row_group(ri)
+ .expect("Unable to read row group");
+ if let Some(sbbf) =
+ row_group_reader.get_column_bloom_filter(column_index)
+ {
+ if row_group.num_rows() >= positive_values.len() as
i64 {
Review Comment:
I changed it to collect all bloom filters and test against values together.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
