alamb commented on code in PR #8854:
URL: https://github.com/apache/arrow-rs/pull/8854#discussion_r2541695529
##########
arrow-buffer/src/buffer/immutable.rs:
##########
@@ -115,6 +114,150 @@ impl Buffer {
Self::from(bytes)
}
+ /// Create a new [`Buffer`] by applying the bitwise operation `op` to two
input buffers.
+ ///
+ /// This function is highly optimized for bitwise operations on large
+ /// bitmaps by processing input buffers in chunks of 64 bits (8 bytes) at a
+ /// time, and thus is much faster than applying the operation bit by bit.
+ ///
+ /// # Notes:
+ /// * `op` takes two `u64` inputs and produces one `u64` output,
+ /// operating on 64 bits at a time. **It must only apply bitwise
operations
+ /// on the relevant bits, as the input `u64` may contain irrelevant bits
+ /// and may be processed differently on different endian architectures.**
+ /// * The inputs are treated as bitmaps, meaning that offsets and length
+ /// are specified in number of bits.
+ /// * The output always has zero offset
+ ///
+ /// # See Also
+ /// - [`Buffer::from_bitwise_unary_op`] for unary operations on a single
input buffer.
+ /// - [`apply_bitwise_binary_op`](bit_util::apply_bitwise_binary_op) for
in-place binary bitwise operations
+ ///
+ /// # Example: Create new [`Buffer`] from bitwise `AND` of two [`Buffer`]s
+ /// ```
+ /// # use arrow_buffer::Buffer;
+ /// let left = Buffer::from(&[0b11001100u8, 0b10111010u8]); // 2 bytes =
16 bits
+ /// let right = Buffer::from(&[0b10101010u8, 0b11011100u8, 0b11110000u8]);
// 3 bytes = 24 bits
+ /// // AND of the first 12 bits
+ /// let result = Buffer::from_bitwise_binary_op(
+ /// &left, 0, &right, 0, 12, |a, b| a & b
+ /// );
+ /// assert_eq!(result.as_slice(), &[0b10001000u8, 0b00001000u8]);
+ /// ```
+ ///
+ /// # Example: Create new [`Buffer`] from bitwise `OR` of two byte slices
+ /// ```
+ /// # use arrow_buffer::Buffer;
+ /// let left = [0b11001100u8, 0b10111010u8];
+ /// let right = [0b10101010u8, 0b11011100u8];
+ /// // OR of bits 4..16 from left and bits 0..12 from right
+ /// let result = Buffer::from_bitwise_binary_op(
+ /// &left, 4, &right, 0, 12, |a, b| a | b
+ /// );
+ /// assert_eq!(result.as_slice(), &[0b10101110u8, 0b00001111u8]);
+ /// ```
+ pub fn from_bitwise_binary_op<F>(
+ left: impl AsRef<[u8]>,
+ left_offset_in_bits: usize,
+ right: impl AsRef<[u8]>,
+ right_offset_in_bits: usize,
+ len_in_bits: usize,
+ mut op: F,
+ ) -> Buffer
+ where
+ F: FnMut(u64, u64) -> u64,
+ {
+ let left_chunks = BitChunks::new(left.as_ref(), left_offset_in_bits,
len_in_bits);
+ let right_chunks = BitChunks::new(right.as_ref(),
right_offset_in_bits, len_in_bits);
+
+ let chunks = left_chunks
+ .iter()
+ .zip(right_chunks.iter())
+ .map(|(left, right)| op(left, right));
+ // Soundness: `BitChunks` is a `BitChunks` iterator which
+ // correctly reports its upper bound
+ let mut buffer = unsafe { MutableBuffer::from_trusted_len_iter(chunks)
};
+
+ let remainder_bytes = ceil(left_chunks.remainder_len(), 8);
+ let rem = op(left_chunks.remainder_bits(),
right_chunks.remainder_bits());
+ // we are counting its starting from the least significant bit, to
to_le_bytes should be correct
+ let rem = &rem.to_le_bytes()[0..remainder_bytes];
+ buffer.extend_from_slice(rem);
Review Comment:
That is a good call -- I will make the change
However, this is same code as how the current bitwise_binary_op does it, so
I would expect no performance difference 🤔
https://github.com/apache/arrow-rs/pull/8854/files#diff-e7a951ab8abfeef1016ed4427a3aef25be5be470454caa1e1dd93e56968316b5L122
why I am susp
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
