This is an automated email from the ASF dual-hosted git repository.
tisonkun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datasketches-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 2b0692e refactor: polish release-facing APIs before 0.5.0 (#256)
2b0692e is described below
commit 2b0692e0d3b818c7f2d5dfcc1a4a1915f213ef42
Author: tison <[email protected]>
AuthorDate: Tue Sep 1 09:06:26 2026 +0800
refactor: polish release-facing APIs before 0.5.0 (#256)
---
CHANGELOG.md | 3 ++-
LICENSE | 7 +++---
NOTICE | 2 --
RELEASE.md | 5 ++++-
datasketches/LICENSE | 1 +
datasketches/NOTICE | 1 +
datasketches/src/countmin/sketch.rs | 23 ++++++++++++++++++-
datasketches/src/cpc/sketch.rs | 26 +++++-----------------
datasketches/src/cpc/wrapper.rs | 10 +++++++--
datasketches/src/frequencies/sketch.rs | 5 +++++
datasketches/src/hll/sketch.rs | 5 +++++
datasketches/src/tdigest/sketch.rs | 18 +++++++++++++++
datasketches/src/thetafamily/theta/intersection.rs | 5 +++++
datasketches/src/thetafamily/theta/sketch.rs | 5 +++++
datasketches/src/thetafamily/theta/union.rs | 5 +++++
datasketches/src/thetafamily/tuple/sketch.rs | 5 +++++
tests-integration/tests/countmin_test/sketch.rs | 11 +++++++++
tests-integration/tests/cpc_test/union.rs | 1 -
tests-integration/tests/cpc_test/update.rs | 3 ---
tests-integration/tests/serde_tests/cpc.rs | 12 ----------
20 files changed, 105 insertions(+), 48 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 89e3b79..332fe0f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,7 +15,7 @@ All significant changes to this project will be documented in
this file.
* `FrequentItemsSketch::new` now rejects map sizes below the minimum of 8
instead of silently rounding them up.
* Replace `FrequentItemsSketch::epsilon_for_lg` with the fallible
`epsilon_for_max_map_size`, and change `apriori_error` to accept the same
maximum map size plus an unsigned stream weight. These helpers now match the
constructor's units, and `max_map_size` exposes the configured value.
* Replace the `is_f32` flag on `TDigestMut::deserialize` with separate
`deserialize` and `deserialize_f32` entry points, making the serialized
precision explicit at the call site.
-* Remove `CpcUnion::num_coupons`, which exposed internal union state solely
for tests. Inspect the resulting `CpcSketch` when diagnostics are needed.
+* Remove `CpcSketch::{validate, num_coupons}` and `CpcUnion::num_coupons`,
which exposed internal state solely for tests. Use cardinality estimates,
confidence bounds, and serialization round trips to inspect observable sketch
behavior.
* Tuple sketch iterators now yield `&TupleEntry<_>` values instead of `(hash,
&summary)` pairs. Use `entry.hash()` and `entry.summary()` to inspect each
retained entry.
* `ThetaIntersection::to_sketch` and `TupleIntersection::to_sketch` now return
`Option`. Callers must handle `None` until the intersection receives its first
successful update.
* `BloomFilterBuilder`, `ThetaSketchBuilder`, `ThetaUnionBuilder`,
`TupleSketchBuilder`, and `TupleUnionBuilder` now validate their configuration
when `build` is called, and `build` returns `Result`. Callers must propagate or
handle construction errors.
@@ -37,6 +37,7 @@ All significant changes to this project will be documented in
this file.
* Bloom filter accuracy construction now rejects targets that exceed the
maximum serialized filter size instead of silently reducing capacity and
violating the requested false-positive probability.
* T-Digest CDF and PMF queries now accept an empty split-point slice and
return the single all-values bin instead of panicking.
* Bloom filter deserialization now rejects malformed images with inconsistent
counts or payload lengths, while valid images with a dirty cached count are
restored correctly.
+* Count-Min deserialization now rejects truncated counter payloads before
allocating the table declared by the image header.
* `FrequentItemsSketch` now enforces the cross-language map-size limit of
`2^30` consistently. Oversized construction returns `InvalidArgument`, and
malformed or oversized serialized images return `InvalidData` instead of
panicking or attempting excessive allocation.
* `FrequentItemsSketch<String>` now rejects an encoded string length that
exceeds the remaining input before allocating the string buffer.
* T-Digest compression now supports `k = u16::MAX` without overflowing.
diff --git a/LICENSE b/LICENSE
index af4da02..1db2a3b 100644
--- a/LICENSE
+++ b/LICENSE
@@ -208,11 +208,10 @@ APPENDIX A: How to apply the Apache License to your work.
APPENDIX B: Additional licenses relevant to this product:
- This product includes a number of source files with code that has been
- adapted from 3rd party sources including sources that may be subject
- to different copyright notices and license terms. Your use of
+ This product includes a number of source files with code that has been
+ adapted from 3rd party sources including sources that may be subject
+ to different copyright notices and license terms. Your use of
the source code for these subcomponents is subject to the terms and
conditions of the following licenses.
(EMPTY)
-
diff --git a/NOTICE b/NOTICE
index 6e2d05e..ecfbe6c 100644
--- a/NOTICE
+++ b/NOTICE
@@ -3,5 +3,3 @@ Copyright 2025-2026 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
-
-
diff --git a/RELEASE.md b/RELEASE.md
index 64f4b01..87ba7f1 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -125,7 +125,10 @@ cargo x prepare-testdata
cargo x lint
cargo x check
cargo x test
-cargo package --list -p datasketches
+package_files="$(cargo package --list -p datasketches)"
+printf '%s\n' "$package_files"
+grep -Fx LICENSE <<<"$package_files"
+grep -Fx NOTICE <<<"$package_files"
cargo publish --dry-run --locked -p datasketches
```
diff --git a/datasketches/LICENSE b/datasketches/LICENSE
new file mode 120000
index 0000000..ea5b606
--- /dev/null
+++ b/datasketches/LICENSE
@@ -0,0 +1 @@
+../LICENSE
\ No newline at end of file
diff --git a/datasketches/NOTICE b/datasketches/NOTICE
new file mode 120000
index 0000000..7e1b82f
--- /dev/null
+++ b/datasketches/NOTICE
@@ -0,0 +1 @@
+../NOTICE
\ No newline at end of file
diff --git a/datasketches/src/countmin/sketch.rs
b/datasketches/src/countmin/sketch.rs
index 484e6c4..7970c0c 100644
--- a/datasketches/src/countmin/sketch.rs
+++ b/datasketches/src/countmin/sketch.rs
@@ -339,6 +339,11 @@ impl<T: CountMinValue> CountMinSketch<T> {
/// Deserializes a sketch from bytes using the default seed.
///
+ /// # Errors
+ ///
+ /// Returns `InvalidData` if the image is malformed or its seed hash does
not match the default
+ /// seed.
+ ///
/// # Examples
///
/// ```
@@ -417,8 +422,24 @@ impl<T: CountMinValue> CountMinSketch<T> {
)?;
let entries = entries_for_config_checked(num_hashes, num_buckets)?;
+ let is_empty = (flags & FLAGS_IS_EMPTY) != 0;
+ if !is_empty {
+ let payload_values = entries
+ .checked_add(1)
+ .ok_or_else(|| Error::deserial("CountMin payload value count
overflows"))?;
+ let payload_bytes = payload_values
+ .checked_mul(LONG_SIZE_BYTES)
+ .ok_or_else(|| Error::deserial("CountMin payload size
overflows"))?;
+ if payload_bytes > cursor.remaining().len() {
+ return Err(Error::insufficient_data(format!(
+ "CountMin payload requires {payload_bytes} bytes, got {}",
+ cursor.remaining().len()
+ )));
+ }
+ }
+
let mut sketch = Self::make(num_hashes, num_buckets, seed,
expected_seed_hash, entries);
- if (flags & FLAGS_IS_EMPTY) != 0 {
+ if is_empty {
return Ok(sketch);
}
diff --git a/datasketches/src/cpc/sketch.rs b/datasketches/src/cpc/sketch.rs
index 8599e46..a9db202 100644
--- a/datasketches/src/cpc/sketch.rs
+++ b/datasketches/src/cpc/sketch.rs
@@ -36,7 +36,6 @@ use crate::cpc::compression::encode_pairs;
use crate::cpc::compression::encode_window;
use crate::cpc::compression_data::COLUMN_PERMUTATIONS_FOR_DECODING;
use crate::cpc::compression_data::COLUMN_PERMUTATIONS_FOR_ENCODING;
-use crate::cpc::count_bits_set_in_matrix;
use crate::cpc::determine_correct_offset;
use crate::cpc::determine_flavor;
use crate::cpc::estimator::estimate;
@@ -606,6 +605,11 @@ impl CpcSketch {
}
/// Deserializes a `CpcSketch` from bytes.
+ ///
+ /// # Errors
+ ///
+ /// Returns `InvalidData` if the image is malformed or its seed hash does
not match the default
+ /// seed.
pub fn deserialize(bytes: &[u8]) -> Result<Self, Error> {
Self::deserialize_with_seed(bytes, DEFAULT_UPDATE_SEED)
}
@@ -926,23 +930,3 @@ impl CpcSketch {
Ok(max_bytes)
}
}
-
-impl CpcSketch {
- /// Returns `true` if the sketch's internal state is valid.
- ///
- /// This is intended for testing and validation purposes.
- #[doc(hidden)]
- pub fn validate(&self) -> bool {
- let bit_matrix = self.build_bit_matrix();
- let num_bits_set = count_bits_set_in_matrix(&bit_matrix);
- num_bits_set == self.num_coupons
- }
-
- /// Returns the number of coupons in the sketch.
- ///
- /// This is intended for testing and validation purposes.
- #[doc(hidden)]
- pub fn num_coupons(&self) -> u32 {
- self.num_coupons
- }
-}
diff --git a/datasketches/src/cpc/wrapper.rs b/datasketches/src/cpc/wrapper.rs
index c0ee945..b32df27 100644
--- a/datasketches/src/cpc/wrapper.rs
+++ b/datasketches/src/cpc/wrapper.rs
@@ -34,7 +34,7 @@ use crate::cpc::serialization::SERIAL_VERSION;
use crate::cpc::serialization::make_preamble_ints;
use crate::error::Error;
-/// A read-only view of a serialized `CpcSketch` image.
+/// Cardinality metadata extracted from a serialized `CpcSketch` image.
#[derive(Debug, Clone)]
pub struct CpcWrapper {
lg_k: u8,
@@ -44,7 +44,13 @@ pub struct CpcWrapper {
}
impl CpcWrapper {
- /// Creates a new `CpcWrapper` from the given byte slice without copying
bytes.
+ /// Reads the cardinality metadata without copying or fully deserializing
the compressed
+ /// payload.
+ ///
+ /// # Errors
+ ///
+ /// Returns `InvalidData` if the preamble is malformed or does not
describe a compressed CPC
+ /// image.
pub fn new(bytes: &[u8]) -> Result<Self, Error> {
let mut cursor = SketchSlice::new(bytes);
let preamble_ints = cursor
diff --git a/datasketches/src/frequencies/sketch.rs
b/datasketches/src/frequencies/sketch.rs
index f96bc90..31383e9 100644
--- a/datasketches/src/frequencies/sketch.rs
+++ b/datasketches/src/frequencies/sketch.rs
@@ -748,6 +748,11 @@ impl<T: FrequentItemValue> FrequentItemsSketch<T> {
/// Deserializes a sketch from bytes.
///
+ /// # Errors
+ ///
+ /// Returns `InvalidData` if the image is truncated, its metadata is
inconsistent, or an item
+ /// cannot be decoded by `T`.
+ ///
/// # Examples
///
/// Built-in support for `i64`:
diff --git a/datasketches/src/hll/sketch.rs b/datasketches/src/hll/sketch.rs
index 4e0ed71..a229b3b 100644
--- a/datasketches/src/hll/sketch.rs
+++ b/datasketches/src/hll/sketch.rs
@@ -294,6 +294,11 @@ impl HllSketch {
/// Deserializes an HLL sketch from bytes.
///
+ /// # Errors
+ ///
+ /// Returns `InvalidData` if the image is truncated or contains an invalid
preamble,
+ /// configuration, or payload.
+ ///
/// # Examples
///
/// ```
diff --git a/datasketches/src/tdigest/sketch.rs
b/datasketches/src/tdigest/sketch.rs
index f55c426..942c735 100644
--- a/datasketches/src/tdigest/sketch.rs
+++ b/datasketches/src/tdigest/sketch.rs
@@ -537,6 +537,11 @@ impl TDigestMut {
/// auto-detected. Use [`deserialize_f32()`](Self::deserialize_f32) for
the compact
/// DataSketches C++ `tdigest<float>` format.
///
+ /// # Errors
+ ///
+ /// Returns `InvalidData` if the image is truncated, has an unsupported
format, or contains
+ /// invalid extrema, centroids, or weights.
+ ///
/// # Examples
///
/// ```
@@ -558,6 +563,11 @@ impl TDigestMut {
/// This format stores centroid means and weights as `(f32, u32)` and is
emitted by the C++
/// `tdigest<float>` implementation. Its header does not identify the
scalar width, so callers
/// must select this entry point explicitly.
+ ///
+ /// # Errors
+ ///
+ /// Returns `InvalidData` if the image is truncated, has an unsupported
format, or contains
+ /// invalid extrema, centroids, or weights.
pub fn deserialize_f32(bytes: &[u8]) -> Result<Self, Error> {
Self::deserialize_impl(bytes, true)
}
@@ -1056,11 +1066,19 @@ impl TDigest {
/// The format of the [reference
implementation](https://github.com/tdunning/t-digest) is
/// auto-detected. Use [`deserialize_f32()`](Self::deserialize_f32) for
the compact
/// DataSketches C++ `tdigest<float>` format.
+ ///
+ /// # Errors
+ ///
+ /// Returns `InvalidData` under the same conditions as
[`TDigestMut::deserialize`].
pub fn deserialize(bytes: &[u8]) -> Result<Self, Error> {
Ok(TDigestMut::deserialize(bytes)?.freeze())
}
/// Deserializes an immutable t-digest from the compact single-precision
DataSketches format.
+ ///
+ /// # Errors
+ ///
+ /// Returns `InvalidData` under the same conditions as
[`TDigestMut::deserialize_f32`].
pub fn deserialize_f32(bytes: &[u8]) -> Result<Self, Error> {
Ok(TDigestMut::deserialize_f32(bytes)?.freeze())
}
diff --git a/datasketches/src/thetafamily/theta/intersection.rs
b/datasketches/src/thetafamily/theta/intersection.rs
index 8f59f9a..ace0d41 100644
--- a/datasketches/src/thetafamily/theta/intersection.rs
+++ b/datasketches/src/thetafamily/theta/intersection.rs
@@ -63,6 +63,11 @@ impl ThetaIntersection {
/// The intersection can be viewed as starting from the "universe" set,
/// and every update can reduce the current set to leave the overlapping
/// subset only.
+ ///
+ /// # Errors
+ ///
+ /// Returns `InvalidArgument` if a non-empty `sketch` has a different seed
hash or its retained
+ /// entries are inconsistent with its metadata.
pub fn update<'a>(&mut self, sketch: impl Into<ThetaSketchView<'a>>) ->
Result<(), Error> {
let sketch = sketch.into();
self.state.update(sketch)
diff --git a/datasketches/src/thetafamily/theta/sketch.rs
b/datasketches/src/thetafamily/theta/sketch.rs
index 88f002e..7fb9ea0 100644
--- a/datasketches/src/thetafamily/theta/sketch.rs
+++ b/datasketches/src/thetafamily/theta/sketch.rs
@@ -686,6 +686,11 @@ impl CompactThetaSketch {
}
/// Deserializes a compact theta sketch from bytes.
+ ///
+ /// # Errors
+ ///
+ /// Returns `InvalidData` if the image is malformed or its seed hash does
not match the default
+ /// seed.
pub fn deserialize(bytes: &[u8]) -> Result<Self, Error> {
Self::deserialize_with_seed(bytes, DEFAULT_UPDATE_SEED)
}
diff --git a/datasketches/src/thetafamily/theta/union.rs
b/datasketches/src/thetafamily/theta/union.rs
index 12dff15..c315c4f 100644
--- a/datasketches/src/thetafamily/theta/union.rs
+++ b/datasketches/src/thetafamily/theta/union.rs
@@ -40,6 +40,11 @@ impl UnionMergePolicy<ThetaEntry> for NoopUnionPolicy {
impl ThetaUnion {
/// Updates this union with the given sketch.
+ ///
+ /// # Errors
+ ///
+ /// Returns `InvalidArgument` if a non-empty `sketch` has a different seed
hash from this
+ /// union.
pub fn update<'a>(&mut self, sketch: impl Into<ThetaSketchView<'a>>) ->
Result<(), Error> {
let sketch = sketch.into();
self.state.update(sketch)
diff --git a/datasketches/src/thetafamily/tuple/sketch.rs
b/datasketches/src/thetafamily/tuple/sketch.rs
index 796e17e..0fddc98 100644
--- a/datasketches/src/thetafamily/tuple/sketch.rs
+++ b/datasketches/src/thetafamily/tuple/sketch.rs
@@ -618,6 +618,11 @@ impl<S> CompactTupleSketch<S> {
}
/// Deserializes a compact Tuple sketch using the default seed.
+ ///
+ /// # Errors
+ ///
+ /// Returns `InvalidData` if the image is malformed, its seed hash does
not match the default
+ /// seed, or a summary cannot be decoded by `S`.
pub fn deserialize(bytes: &[u8]) -> Result<Self, Error>
where
S: TupleSummaryValue,
diff --git a/tests-integration/tests/countmin_test/sketch.rs
b/tests-integration/tests/countmin_test/sketch.rs
index d22cf30..42f8530 100644
--- a/tests-integration/tests/countmin_test/sketch.rs
+++ b/tests-integration/tests/countmin_test/sketch.rs
@@ -259,6 +259,17 @@ fn test_serialize_deserialize_non_empty_u64() {
assert_eq!(decoded.estimate(42u64), sketch.estimate(42u64));
}
+#[test]
+fn test_truncated_non_empty_payload_is_rejected_before_table_allocation() {
+ let mut bytes = CountMinSketch::<i64>::new(1, 3).unwrap().serialize();
+ bytes[3] = 0;
+ bytes[8..12].copy_from_slice(&(1u32 << 29).to_le_bytes());
+
+ let error = CountMinSketch::<i64>::deserialize(&bytes).unwrap_err();
+ assert_eq!(error.kind(), ErrorKind::InvalidData);
+ assert!(error.message().contains("payload requires"));
+}
+
#[test]
fn test_invalid_hashes_return_error() {
let error = CountMinSketch::<i64>::new(0, 5).unwrap_err();
diff --git a/tests-integration/tests/cpc_test/union.rs
b/tests-integration/tests/cpc_test/union.rs
index 9180165..7b28034 100644
--- a/tests-integration/tests/cpc_test/union.rs
+++ b/tests-integration/tests/cpc_test/union.rs
@@ -97,7 +97,6 @@ fn test_sliding_union_matches_single_sketch() {
}
let result = union.to_sketch();
assert!(!result.is_empty());
- assert!(result.num_coupons() >= 27 * (1 << 11) / 8);
let estimate = sketch.estimate();
assert_that!(
result.estimate(),
diff --git a/tests-integration/tests/cpc_test/update.rs
b/tests-integration/tests/cpc_test/update.rs
index ab806f6..99a1cb5 100644
--- a/tests-integration/tests/cpc_test/update.rs
+++ b/tests-integration/tests/cpc_test/update.rs
@@ -32,7 +32,6 @@ fn test_empty() {
assert_eq!(sketch.estimate(), 0.0);
assert_eq!(sketch.lower_bound(NumStdDev::One), 0.0);
assert_eq!(sketch.upper_bound(NumStdDev::One), 0.0);
- assert!(sketch.validate());
}
#[test]
@@ -43,7 +42,6 @@ fn test_one_value() {
assert_eq!(sketch.estimate(), 1.0);
assert_that!(sketch.estimate(), ge(sketch.lower_bound(NumStdDev::One)));
assert_that!(sketch.estimate(), le(sketch.upper_bound(NumStdDev::One)));
- assert!(sketch.validate());
}
#[test]
@@ -59,7 +57,6 @@ fn test_many_values() {
);
assert_that!(sketch.estimate(), ge(sketch.lower_bound(NumStdDev::One)));
assert_that!(sketch.estimate(), le(sketch.upper_bound(NumStdDev::One)));
- assert!(sketch.validate());
}
#[test]
diff --git a/tests-integration/tests/serde_tests/cpc.rs
b/tests-integration/tests/serde_tests/cpc.rs
index cf27650..78c61fd 100644
--- a/tests-integration/tests/serde_tests/cpc.rs
+++ b/tests-integration/tests/serde_tests/cpc.rs
@@ -59,29 +59,17 @@ fn test_sketch_file(path: &Path, expected_cardinality:
usize) -> CpcSketch {
fn test_sketch_replay(path: &Path, sketch: CpcSketch, inputs: impl
Iterator<Item = usize>) {
let initial_estimate = sketch.estimate();
- let initial_num_coupons = sketch.num_coupons();
let mut sketch = sketch;
for value in inputs {
sketch.update(value);
}
- assert_eq!(
- initial_num_coupons,
- sketch.num_coupons(),
- "Coupon count changed after replaying input for {}",
- path.display()
- );
assert_eq!(
initial_estimate,
sketch.estimate(),
"Estimate changed after replaying input for {}",
path.display()
);
- assert!(
- sketch.validate(),
- "Sketch became invalid after replaying input for {}",
- path.display()
- );
}
#[test]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
