adriangb commented on code in PR #11641:
URL: https://github.com/apache/arrow-rs/pull/11641#discussion_r4568121802
##########
parquet/src/bloom_filter/mod.rs:
##########
@@ -857,6 +931,65 @@ fn read_bloom_filter_header_and_length(buffer: &[u8]) ->
Result<(BloomFilterHead
mod tests {
use super::*;
+ fn bitset(f: &Sbbf) -> Vec<u8> {
+ let mut out = vec![];
+ f.write_bitset(&mut out).unwrap();
+ out
+ }
+
+ #[test]
+ fn test_branch_free_check_matches_check() {
+ let mut filter = Sbbf::new_with_num_of_bytes(256);
+ for i in 0..200u64 {
+ filter.insert(&(i * 7));
+ }
+ for i in 0..5000u64 {
+ let h = hash_as_bytes(&i);
+ assert_eq!(filter.check_hash(h), filter.check_hash_branch_free(h));
+ }
+ }
+
+ #[cfg(feature = "arrow")]
+ #[test]
+ fn test_check_hashes_matches_check_hash() {
+ let hashes: Vec<u64> = (0..5000u64).map(|i|
hash_as_bytes(&i)).collect();
+ let mut filter = Sbbf::new_with_num_of_bytes(256);
+ filter.insert_hashes(&hashes[..200]);
+ let result = filter.check_hashes(&hashes);
+ for (i, &h) in hashes.iter().enumerate() {
+ assert_eq!(result.value(i), filter.check_hash(h));
+ }
+ assert!((0..200).all(|i| result.value(i)));
+ }
+
+ #[test]
+ fn test_union_of_parts_matches_one_filter() {
+ let hashes: Vec<u64> = (0..100_000u64).map(|i|
hash_as_bytes(&i)).collect();
+ let mut whole = Sbbf::new_with_num_of_bytes(1 << 20);
+ whole.insert_hashes(&hashes);
+ let mut merged = Sbbf::new_with_num_of_bytes(1 << 20);
+ for chunk in hashes.chunks(7_000) {
+ let mut part = Sbbf::new_with_num_of_bytes(1 << 20);
+ part.insert_hashes(chunk);
+ merged.union(&part).unwrap();
+ }
+ assert_eq!(bitset(&whole), bitset(&merged));
+ // union then fold == fold then union
+ let mut folded = whole.clone();
+ folded.fold_to_target_fpp(0.01);
+ merged.fold_to_target_fpp(0.01);
+ assert_eq!(bitset(&folded), bitset(&merged));
Review Comment:
Done in 314d407c7018af3f6ea32e228bae7599207ce9cc
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]