This is an automated email from the ASF dual-hosted git repository.
tisonkun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datasketches-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 90fbb6b refactor(xor): make XorFilter::serialize return Vec<u8> (#347)
90fbb6b is described below
commit 90fbb6bccf0fded4524181ef94f1e2bef4dfc9d5
Author: tison <[email protected]>
AuthorDate: Thu Nov 12 01:17:31 2026 +0800
refactor(xor): make XorFilter::serialize return Vec<u8> (#347)
---
datasketches/src/xor/filter.rs | 41 ++++++++++--------------------
tests-integration/tests/serde_tests/xor.rs | 25 ++++++++----------
2 files changed, 23 insertions(+), 43 deletions(-)
diff --git a/datasketches/src/xor/filter.rs b/datasketches/src/xor/filter.rs
index 8cd8c1f..53c270f 100644
--- a/datasketches/src/xor/filter.rs
+++ b/datasketches/src/xor/filter.rs
@@ -80,14 +80,6 @@ enum Fingerprints {
}
impl Fingerprints {
- fn len(&self) -> usize {
- match self {
- Self::Xor8(values) => values.len(),
- Self::Xor16(values) => values.len(),
- Self::Xor32(values) => values.len(),
- }
- }
-
fn byte_len(&self) -> usize {
match self {
Self::Xor8(values) => size_of_val::<[u8]>(values),
@@ -320,22 +312,8 @@ impl XorFilter {
}
/// Serializes the filter in the DataSketches format.
- ///
- /// # Errors
- ///
- /// Returns [`InvalidArgument`](crate::error::ErrorKind::InvalidArgument)
if the fingerprint
- /// slot count exceeds `i32::MAX` or the serialized size overflows
`usize`. Construction allows
- /// larger capacities than the standard format.
- pub fn serialize(&self) -> Result<Vec<u8>, Error> {
- if self.fingerprints.len() > i32::MAX as usize {
- return Err(Error::invalid_argument(
- "xor filter exceeds standard format limits",
- ));
- }
- let size = PREAMBLE_BYTES
- .checked_add(self.fingerprints.byte_len())
- .ok_or_else(|| Error::invalid_argument("xor filter serialized size
overflow"))?;
- let mut bytes = SketchBytes::with_capacity(size);
+ pub fn serialize(&self) -> Vec<u8> {
+ let mut bytes = SketchBytes::with_capacity(PREAMBLE_BYTES +
self.fingerprints.byte_len());
bytes.write_u8(Family::XORFILTER.min_pre_longs);
bytes.write_u8(SERIAL_VERSION);
@@ -361,7 +339,7 @@ impl XorFilter {
}
}
- Ok(bytes.into_bytes())
+ bytes.into_bytes()
}
/// Deserializes an owned filter from bytes.
@@ -490,7 +468,7 @@ fn hash_item<T: Hash + ?Sized>(item: &T) -> u64 {
}
fn compute_capacity(num_items: usize) -> Result<usize, Error> {
- if num_items > u32::MAX as usize {
+ if num_items > i32::MAX as usize {
return Err(Error::invalid_argument(format!(
"xor filter item count exceeds construction limit: {num_items}"
)));
@@ -499,7 +477,7 @@ fn compute_capacity(num_items: usize) -> Result<usize,
Error> {
let scaled = (LOAD_FACTOR * num_items as f64) as u64;
let capacity = CAPACITY_OFFSET + scaled;
let capacity = capacity / u64::from(NUM_HASHES) * u64::from(NUM_HASHES);
- if capacity > u32::MAX as u64 || capacity > (isize::MAX as u64 /
size_of::<u64>() as u64) {
+ if capacity > i32::MAX as u64 || capacity > (isize::MAX as u64 /
size_of::<u64>() as u64) {
return Err(Error::invalid_argument(format!(
"xor filter capacity exceeds construction limit: {capacity}"
)));
@@ -683,12 +661,19 @@ mod tests {
#[test]
fn oversized_input_is_rejected_before_allocating() {
- for count in [usize::MAX, u32::MAX as usize] {
+ for count in [
+ usize::MAX,
+ u32::MAX as usize,
+ i32::MAX as usize,
+ 1_745_921_640,
+ ] {
assert_eq!(
compute_capacity(count).unwrap_err().kind(),
ErrorKind::InvalidArgument
);
}
+ #[cfg(target_pointer_width = "64")]
+ assert_eq!(compute_capacity(1_745_921_639).unwrap(), 2_147_483_646);
}
#[test]
diff --git a/tests-integration/tests/serde_tests/xor.rs
b/tests-integration/tests/serde_tests/xor.rs
index 2fe4d15..684fcd5 100644
--- a/tests-integration/tests/serde_tests/xor.rs
+++ b/tests-integration/tests/serde_tests/xor.rs
@@ -46,8 +46,8 @@ fn java_images_round_trip_byte_for_byte() {
let built = builder.build().unwrap();
let restored = XorFilter::deserialize(&expected).unwrap();
- assert_eq!(restored.serialize().unwrap(), expected);
- assert_eq!(built.serialize().unwrap(), expected);
+ assert_eq!(restored.serialize(), expected);
+ assert_eq!(built.serialize(), expected);
for value in values {
assert!(built.contains(&value));
assert!(
@@ -68,7 +68,7 @@ fn serialization_round_trips_standard_fingerprint_types() {
for count in [0, 1, 10_000_u64] {
let hashes: Vec<_> = (0..count).collect();
let original = XorFilter::from_hashes(&hashes, ty, 0).unwrap();
- let bytes = original.serialize().unwrap();
+ let bytes = original.serialize();
let restored = XorFilter::deserialize(&bytes).unwrap();
assert_eq!(restored, original);
@@ -87,10 +87,7 @@ fn every_truncated_image_is_rejected() {
XorFilterType::Xor32,
] {
let hashes: Vec<_> = (0..100_u64).collect();
- let bytes = XorFilter::from_hashes(&hashes, ty, 0)
- .unwrap()
- .serialize()
- .unwrap();
+ let bytes = XorFilter::from_hashes(&hashes, ty,
0).unwrap().serialize();
for end in 0..bytes.len() {
let error = XorFilter::deserialize(&bytes[..end]).unwrap_err();
assert_eq!(error.kind(), ErrorKind::InvalidData, "length {end}");
@@ -104,8 +101,7 @@ fn invalid_preamble_fields_are_rejected() {
let hashes: Vec<_> = (0..100_u64).collect();
let valid = XorFilter::from_hashes(&hashes, XorFilterType::Xor8, 0)
.unwrap()
- .serialize()
- .unwrap();
+ .serialize();
for (offset, value) in [
(0, 1),
@@ -130,8 +126,7 @@ fn unsafe_lengths_are_rejected_before_indexing() {
let hashes: Vec<_> = (0..100_u64).collect();
let valid = XorFilter::from_hashes(&hashes, XorFilterType::Xor8, 0)
.unwrap()
- .serialize()
- .unwrap();
+ .serialize();
for segment_length in [0_i32, -1, i32::MAX] {
let mut corrupted = valid.clone();
@@ -152,7 +147,7 @@ fn trailing_storage_is_ignored() {
] {
let hashes: Vec<_> = (0..100_u64).collect();
let original = XorFilter::from_hashes(&hashes, kind, 0).unwrap();
- let mut bytes = original.serialize().unwrap();
+ let mut bytes = original.serialize();
bytes.extend_from_slice(&[0xaa; 16]);
assert_eq!(XorFilter::deserialize(&bytes).unwrap(), original);
@@ -193,8 +188,8 @@ fn java_utf8_and_raw_byte_inputs_match() {
let built = builder.build().unwrap();
let expected = decode_hex(image);
let restored = XorFilter::deserialize(&expected).unwrap();
- assert_eq!(restored.serialize().unwrap(), expected);
- assert_eq!(built.serialize().unwrap(), expected);
+ assert_eq!(restored.serialize(), expected);
+ assert_eq!(built.serialize(), expected);
for bytes in [b"hello".as_slice(), "数据🔎".as_bytes(), &[], &[0, 1,
255]] {
assert!(built.contains(&raw_bytes::from_slice(bytes)));
assert!(restored.contains(&raw_bytes::from_slice(bytes)));
@@ -214,5 +209,5 @@ fn xor32_image_preserves_full_width_little_endian_slots() {
));
let filter = XorFilter::deserialize(&bytes).unwrap();
assert!(filter.contains_hash(0));
- assert_eq!(filter.serialize().unwrap(), bytes);
+ assert_eq!(filter.serialize(), bytes);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]