This is an automated email from the ASF dual-hosted git repository.
tisonkun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datasketches-rust.git
The following commit(s) were added to refs/heads/main by this push:
new cafebaa fix(frequencies): write the full empty preamble long in
serialize (#143)
cafebaa is described below
commit cafebaae65286bb46df579f0d2425f911184f39e
Author: kedom <[email protected]>
AuthorDate: Fri Jul 3 19:15:59 2026 +0200
fix(frequencies): write the full empty preamble long in serialize (#143)
An empty FrequentItemsSketch serialized to 6 bytes: the preamble
declares one 8-byte preamble long, but serialization stopped after the
flags byte. Deserialization (in Rust as well as the Java and C++
readers) consumes the full 8-byte preamble, so a serialized empty
sketch failed to deserialize with an insufficient-data error.
Emptiness is reachable from non-trivial streams: a map saturated with
count-1 items purges every counter, so pipelines that persist sketches
can hit the failure on real data.
Write the two unused padding bytes so the empty encoding is one full
preamble long, matching the Java and C++ implementations.
Co-authored-by: Claude Fable 5 <[email protected]>
---
CHANGELOG.md | 4 ++++
datasketches/src/frequencies/sketch.rs | 3 ++-
.../tests/frequencies_serialization_test.rs | 25 ++++++++++++++++++++++
3 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 600247b..5f28eda 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,10 @@ All significant changes to this project will be documented in
this file.
* `FrequentItemsSketch` now supports borrowed-key updates via `update_ref` and
`update_with_count_ref`, allowing sketches such as
`FrequentItemsSketch<String>` to update from `&str` without allocating on
existing-key hits. Frequency queries also accept borrowed key forms matching
`Borrow<Q>`.
+### Bug fixes
+
+* `FrequentItemsSketch::serialize` now writes the full 8-byte preamble for an
empty sketch, matching the Java and C++ encoding. Empty sketches previously
serialized to 6 bytes, which `FrequentItemsSketch::deserialize` rejected with
an insufficient-data error.
+
## v0.3.0 (2026-05-18)
### Breaking changes
diff --git a/datasketches/src/frequencies/sketch.rs
b/datasketches/src/frequencies/sketch.rs
index f46a231..cf9f06d 100644
--- a/datasketches/src/frequencies/sketch.rs
+++ b/datasketches/src/frequencies/sketch.rs
@@ -478,13 +478,14 @@ impl<T: Eq + Hash> FrequentItemsSketch<T> {
T: Clone, // for self.hash_map.active_keys()
{
if self.is_empty() {
- let mut bytes = SketchBytes::with_capacity(8);
+ let mut bytes = SketchBytes::with_capacity(PREAMBLE_LONGS_EMPTY as
usize * 8);
bytes.write_u8(PREAMBLE_LONGS_EMPTY);
bytes.write_u8(SERIAL_VERSION);
bytes.write_u8(Family::FREQUENCY.id);
bytes.write_u8(self.lg_max_map_size);
bytes.write_u8(self.hash_map.lg_length());
bytes.write_u8(EMPTY_FLAG_MASK);
+ bytes.write_u16_le(0); // unused
return bytes.into_bytes();
}
diff --git a/datasketches/tests/frequencies_serialization_test.rs
b/datasketches/tests/frequencies_serialization_test.rs
index fd82761..f070e91 100644
--- a/datasketches/tests/frequencies_serialization_test.rs
+++ b/datasketches/tests/frequencies_serialization_test.rs
@@ -52,6 +52,31 @@ fn test_items_round_trip() {
assert_eq!(restored.maximum_error(), sketch.maximum_error());
}
+#[test]
+fn test_empty_round_trip() {
+ let sketch = FrequentItemsSketch::<i64>::new(32);
+ let bytes = sketch.serialize();
+ // One preamble long, matching the Java and C++ empty encoding.
+ assert_eq!(bytes.len(), 8);
+ let restored = FrequentItemsSketch::<i64>::deserialize(&bytes).unwrap();
+ assert!(restored.is_empty());
+ assert_eq!(restored.total_weight(), 0);
+ assert_eq!(restored.maximum_error(), 0);
+}
+
+#[test]
+fn test_purged_to_empty_round_trip() {
+ // Saturating the map with count-1 items makes the purge median 1, which
+ // removes every counter and leaves a non-trivial sketch empty.
+ let mut sketch = FrequentItemsSketch::<i64>::new(32);
+ for i in 0..=(32 * 3 / 4) {
+ sketch.update(i);
+ }
+ assert!(sketch.is_empty());
+ let restored =
FrequentItemsSketch::<i64>::deserialize(&sketch.serialize()).unwrap();
+ assert!(restored.is_empty());
+}
+
#[test]
fn test_java_frequent_longs_compatibility() {
let test_cases = [0, 1, 10, 100, 1000, 10000, 100000, 1000000];
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
