viirya commented on code in PR #987:
URL: https://github.com/apache/datafusion-comet/pull/987#discussion_r1803819107
##########
native/core/src/execution/datafusion/util/spark_bloom_filter.rs:
##########
@@ -54,6 +78,25 @@ impl SparkBloomFilter {
num_hash_functions: num_hash_functions as u32,
}
}
+}
+
+impl SparkBloomFilter {
+ /// Serializes a SparkBloomFilter to a byte array conforming to Spark's
BloomFilter
+ /// binary format version 1.
+ pub fn spark_serialization(&self) -> Vec<u8> {
+ // There might be a more efficient way to do this, even with all the
endianness stuff.
+ let mut spark_bloom_filter: Vec<u8> = 1_u32.to_be_bytes().to_vec();
+ spark_bloom_filter.append(&mut
self.num_hash_functions.to_be_bytes().to_vec());
+ spark_bloom_filter.append(&mut (self.bits.word_size() as
u32).to_be_bytes().to_vec());
+ let mut filter_state: Vec<u64> = self.bits.data();
+ for i in filter_state.iter_mut() {
+ *i = i.to_be();
+ }
+ // Does it make sense to do a std::mem::take of filter_state here?
Unclear to me if a deep
+ // copy of filter_state as a Vec<u64> to a Vec<u8> is happening here.
Review Comment:
You mean if `std::mem::take` also does copy?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
