This is an automated email from the ASF dual-hosted git repository.
tisonkun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datasketches-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 5f68589 feat: add Rust serialization snapshot generator (#382)
5f68589 is described below
commit 5f6858906ad153a3029098a91faf59343268258c
Author: tison <[email protected]>
AuthorDate: Wed Nov 18 17:08:41 2026 +0800
feat: add Rust serialization snapshot generator (#382)
---
CHANGELOG.md | 4 +
CONTRIBUTING.md | 17 +-
Cargo.lock | 8 +
Cargo.toml | 1 +
datagen/Cargo.toml | 43 +++++
datagen/src/bloom.rs | 46 +++++
datagen/src/countmin.rs | 52 ++++++
datagen/src/cpc.rs | 46 +++++
datagen/src/density.rs | 54 ++++++
datagen/src/frequencies.rs | 100 ++++++++++
datagen/src/hll.rs | 66 +++++++
datagen/src/kll.rs | 71 +++++++
datagen/src/main.rs | 117 ++++++++++++
datagen/src/req.rs | 66 +++++++
datagen/src/tdigest.rs | 39 ++++
datagen/src/theta.rs | 63 +++++++
datagen/src/tuple.rs | 42 +++++
datagen/src/xor.rs | 48 +++++
datasketches/src/hll/serialization.rs | 16 +-
tests-integration/tests/serde_tests/.gitignore | 1 +
tests-integration/tests/serde_tests/bloom.rs | 40 ++++
tests-integration/tests/serde_tests/countmin.rs | 69 +++++++
tests-integration/tests/serde_tests/cpc.rs | 34 ++++
tests-integration/tests/serde_tests/density.rs | 124 ++++++++++++
tests-integration/tests/serde_tests/frequencies.rs | 150 +++++++++++++++
tests-integration/tests/serde_tests/hll.rs | 80 ++++++++
tests-integration/tests/serde_tests/kll.rs | 80 ++++++++
tests-integration/tests/serde_tests/req.rs | 102 ++++++++++
tests-integration/tests/serde_tests/tdigest.rs | 9 +
tests-integration/tests/serde_tests/theta.rs | 64 +++++++
tests-integration/tests/serde_tests/tuple.rs | 13 ++
tests-integration/tests/serde_tests/xor.rs | 48 +++++
xtask/src/main.rs | 124 +-----------
xtask/src/prepare_testdata.rs | 208 +++++++++++++++++++++
34 files changed, 1913 insertions(+), 132 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7758cbf..289c576 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,10 @@ All significant changes to this project will be documented in
this file.
* Add `DensitySketch`, `Kernel`, `GaussianKernel`, `DensityValue`, and
`DensitySketchIter` behind the `density` feature for streaming kernel density
estimation and coreset construction over multidimensional `f32` and `f64`
vectors, with custom kernel support, merging, iteration, and cross-language
serialization.
+### Bug fixes
+
+* `HllSketch::serialize` and `HllUnion::serialize` now set the compact flag in
serialized `HllType::Hll6` and `HllType::Hll8` HLL-mode headers, matching the
compact wire format expected by cross-language DataSketches readers;
deserialization continues to accept images with or without the flag.
+
## v0.7.0
### Breaking changes
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 50b1ded..b49d8e5 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -174,23 +174,32 @@ cargo install taplo-cli typos-cli hawkeye
## Serialization snapshots
-Serialization compatibility tests use snapshots from a pinned revision of
[`apache/datasketches-tck`](https://github.com/apache/datasketches-tck).
+Serialization compatibility tests use cross-language snapshots from a pinned
revision of
[`apache/datasketches-tck`](https://github.com/apache/datasketches-tck)
together with locally generated Rust snapshots from the workspace `datagen`
binary crate.
-The `cargo x prepare-testdata` command downloads the TCK archive and
synchronizes `.sk` files from
`serialization_test_data/<language>_generated_files` into:
+The `cargo x prepare-testdata` command downloads the TCK archive for remote
languages, runs the local Rust snapshot generator, and populates `.sk` files in:
- `tests-integration/tests/serde_tests/cpp_generated_files`
- `tests-integration/tests/serde_tests/go_generated_files`
- `tests-integration/tests/serde_tests/java_generated_files`
+- `tests-integration/tests/serde_tests/rust_generated_files`
-You can synchronize them separately:
+You can prepare each language separately:
```shell
cargo x prepare-testdata cpp
cargo x prepare-testdata go
cargo x prepare-testdata java
+cargo x prepare-testdata rust
```
-If no language is specified, all languages are prepared. These directories are
not stored in Git. Run the command before the first test run and again whenever
the pinned TCK revision changes. It requires network access and replaces the
selected generated directories.
+If no language is specified, all languages are prepared. These directories are
not stored in Git and are replaced whenever selected. Run the command before
the first test run and again whenever the pinned TCK revision or Rust
serialization behavior changes. Preparing `cpp`, `go`, or `java` requires
network access, while `cargo x prepare-testdata rust` runs offline.
+
+To generate Rust snapshots directly (writing to
`serialization_test_data/rust_generated_files` by default) or filter by sketch
family, run the `datagen` package:
+
+```shell
+cargo run --release --package datagen
+cargo run --release --package datagen -- --output-dir target/snapshots
--sketch hll --sketch theta
+```
## Code of Conduct
diff --git a/Cargo.lock b/Cargo.lock
index 2aee820..8fa0047 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -232,6 +232,14 @@ dependencies = [
"libc",
]
+[[package]]
+name = "datagen"
+version = "0.0.0"
+dependencies = [
+ "clap",
+ "datasketches",
+]
+
[[package]]
name = "datasketches"
version = "0.7.0"
diff --git a/Cargo.toml b/Cargo.toml
index ae16d63..44ca69e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,6 +18,7 @@
[workspace]
members = [
"benchmarks",
+ "datagen",
"datasketches",
"examples",
"tests-integration",
diff --git a/datagen/Cargo.toml b/datagen/Cargo.toml
new file mode 100644
index 0000000..1cc64fb
--- /dev/null
+++ b/datagen/Cargo.toml
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "datagen"
+publish = false
+
+edition.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+clap = { workspace = true }
+datasketches = { workspace = true, features = [
+ "bloom",
+ "countmin",
+ "cpc",
+ "density",
+ "frequencies",
+ "hll",
+ "kll",
+ "req",
+ "tdigest",
+ "theta",
+ "tuple",
+ "xor",
+] }
+
+[lints]
+workspace = true
diff --git a/datagen/src/bloom.rs b/datagen/src/bloom.rs
new file mode 100644
index 0000000..da7f795
--- /dev/null
+++ b/datagen/src/bloom.rs
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+use std::path::Path;
+
+use datasketches::bloom::BloomFilterBuilder;
+use datasketches::hash::value::canonical_float;
+
+use crate::write_snapshot;
+
+pub fn generate(output_dir: &Path) -> io::Result<()> {
+ for n in [0_u64, 10_000, 2_000_000, 30_000_000] {
+ for h in [3_u16, 5] {
+ let mut filter = BloomFilterBuilder::with_size(n.max(1_000), h)
+ .build()
+ .unwrap();
+ for i in 0..(n / 10) {
+ filter.insert(i as i64);
+ }
+ if n > 0 {
+ filter.insert(canonical_float::from_f64(f64::NAN));
+ }
+ write_snapshot(
+ output_dir,
+ &format!("bf_n{n}_h{h}_rust.sk"),
+ &filter.serialize(),
+ )?;
+ }
+ }
+ Ok(())
+}
diff --git a/datagen/src/countmin.rs b/datagen/src/countmin.rs
new file mode 100644
index 0000000..744f413
--- /dev/null
+++ b/datagen/src/countmin.rs
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+use std::path::Path;
+
+use datasketches::countmin::CountMinSketch;
+use datasketches::hash::value::raw_bytes;
+
+use crate::write_snapshot;
+
+pub fn generate(output_dir: &Path) -> io::Result<()> {
+ for n in [0_u64, 1, 10, 100, 1_000, 10_000] {
+ let mut sketch = CountMinSketch::<i64>::new(3, 1_000).unwrap();
+ populate(&mut sketch, n);
+ write_snapshot(
+ output_dir,
+ &format!("count_min_v2_n{n}_rust.sk"),
+ &sketch.serialize(),
+ )?;
+ }
+
+ let mut seeded = CountMinSketch::<i64>::with_seed(3, 1_000, 42).unwrap();
+ populate(&mut seeded, 1_000);
+ write_snapshot(
+ output_dir,
+ "count_min_v2_seed42_n1000_rust.sk",
+ &seeded.serialize(),
+ )?;
+ Ok(())
+}
+
+fn populate(sketch: &mut CountMinSketch<i64>, n: u64) {
+ for i in 0..n {
+ sketch.update_with_weight(i as i64, (i + 1) as i64);
+ sketch.update_with_weight(raw_bytes::from_string(i.to_string()),
1_i64);
+ }
+}
diff --git a/datagen/src/cpc.rs b/datagen/src/cpc.rs
new file mode 100644
index 0000000..98c1e2b
--- /dev/null
+++ b/datagen/src/cpc.rs
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+use std::path::Path;
+
+use datasketches::cpc::CpcSketch;
+use datasketches::hash::value::sign_extend;
+
+use crate::write_snapshot;
+
+pub fn generate(output_dir: &Path) -> io::Result<()> {
+ for n in [0_u64, 100, 200, 2_000, 20_000] {
+ let mut sketch = CpcSketch::new(11).unwrap();
+ for i in 0..n {
+ sketch.update(i as i64);
+ }
+ write_snapshot(
+ output_dir,
+ &format!("cpc_n{n}_rust.sk"),
+ &sketch.serialize(),
+ )?;
+ }
+
+ let mut sketch = CpcSketch::new(11).unwrap();
+ sketch.update(sign_extend::from_i8(-1));
+ sketch.update(sign_extend::from_i16(-1));
+ sketch.update(sign_extend::from_i32(-1));
+ sketch.update(-1_i64);
+ write_snapshot(output_dir, "cpc_negative_one_rust.sk",
&sketch.serialize())?;
+ Ok(())
+}
diff --git a/datagen/src/density.rs b/datagen/src/density.rs
new file mode 100644
index 0000000..035eae6
--- /dev/null
+++ b/datagen/src/density.rs
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+use std::path::Path;
+
+use datasketches::density::DensitySketch;
+
+use crate::write_snapshot;
+
+const COUNTS: [u64; 6] = [0, 1, 10, 100, 1_000, 10_000];
+
+pub fn generate(output_dir: &Path) -> io::Result<()> {
+ for n in COUNTS {
+ let mut sketch = DensitySketch::<f32>::new(10, 3).unwrap();
+ for i in 0..n {
+ let x = i as f32;
+ sketch.update(&[x, x.sqrt(), -x]).unwrap();
+ }
+ write_snapshot(
+ output_dir,
+ &format!("density_float_n{n}_rust.sk"),
+ &sketch.serialize(),
+ )?;
+ }
+
+ for n in COUNTS {
+ let mut sketch = DensitySketch::<f64>::new(10, 3).unwrap();
+ for i in 0..n {
+ let x = i as f64;
+ sketch.update(&[x, x.sqrt(), -x]).unwrap();
+ }
+ write_snapshot(
+ output_dir,
+ &format!("density_double_n{n}_rust.sk"),
+ &sketch.serialize(),
+ )?;
+ }
+ Ok(())
+}
diff --git a/datagen/src/frequencies.rs b/datagen/src/frequencies.rs
new file mode 100644
index 0000000..f852963
--- /dev/null
+++ b/datagen/src/frequencies.rs
@@ -0,0 +1,100 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+use std::path::Path;
+
+use datasketches::frequencies::FrequentItemsSketch;
+
+use crate::N_STD;
+use crate::write_snapshot;
+
+pub fn generate(output_dir: &Path) -> io::Result<()> {
+ for n in N_STD {
+ let mut sketch = FrequentItemsSketch::<i64>::new(64).unwrap();
+ for i in 1..=n {
+ sketch.update(i as i64);
+ }
+ write_snapshot(
+ output_dir,
+ &format!("frequent_long_n{n}_rust.sk"),
+ &sketch.serialize(),
+ )?;
+ }
+
+ for n in N_STD {
+ let mut sketch = FrequentItemsSketch::<String>::new(64).unwrap();
+ for i in 1..=n {
+ sketch.update(i.to_string());
+ }
+ write_snapshot(
+ output_dir,
+ &format!("frequent_string_n{n}_rust.sk"),
+ &sketch.serialize(),
+ )?;
+ }
+
+ let mut long_purged = FrequentItemsSketch::<i64>::new(256).unwrap();
+ for i in 1..=193_i64 {
+ long_purged.update(i);
+ }
+ write_snapshot(
+ output_dir,
+ "frequent_long_purged_rust.sk",
+ &long_purged.serialize(),
+ )?;
+
+ let mut string_purged = FrequentItemsSketch::<String>::new(256).unwrap();
+ for i in 1..=193 {
+ string_purged.update(i.to_string());
+ }
+ write_snapshot(
+ output_dir,
+ "frequent_string_purged_rust.sk",
+ &string_purged.serialize(),
+ )?;
+
+ let mut ascii = FrequentItemsSketch::<String>::new(64).unwrap();
+ for (ch, weight) in [('a', 1_u64), ('b', 2), ('c', 3), ('d', 4)] {
+ ascii.update_with_weight(ch.to_string().repeat(29), weight);
+ }
+ write_snapshot(
+ output_dir,
+ "frequent_string_ascii_rust.sk",
+ &ascii.serialize(),
+ )?;
+
+ let mut utf8 = FrequentItemsSketch::<String>::new(64).unwrap();
+ for (item, weight) in [
+ ("абвгд", 1_u64),
+ ("еёжзи", 2),
+ ("йклмн", 3),
+ ("опрст", 4),
+ ("уфхцч", 5),
+ ("шщъыь", 6),
+ ("эюя", 7),
+ ] {
+ utf8.update_with_weight(item.to_string(), weight);
+ }
+ write_snapshot(
+ output_dir,
+ "frequent_string_utf8_rust.sk",
+ &utf8.serialize(),
+ )?;
+
+ Ok(())
+}
diff --git a/datagen/src/hll.rs b/datagen/src/hll.rs
new file mode 100644
index 0000000..c2d9e1e
--- /dev/null
+++ b/datagen/src/hll.rs
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+use std::path::Path;
+
+use datasketches::hll::HllSketch;
+use datasketches::hll::HllType;
+use datasketches::hll::HllUnion;
+
+use crate::N_STD;
+use crate::write_snapshot;
+
+const TYPES: [(HllType, &str); 3] = [
+ (HllType::Hll4, "hll4"),
+ (HllType::Hll6, "hll6"),
+ (HllType::Hll8, "hll8"),
+];
+
+pub fn generate(output_dir: &Path) -> io::Result<()> {
+ for (hll_type, prefix) in TYPES {
+ for n in N_STD {
+ let mut sketch = HllSketch::new(12, hll_type).unwrap();
+ for i in 0..n {
+ sketch.update(i as i64);
+ }
+ write_snapshot(
+ output_dir,
+ &format!("{prefix}_n{n}_rust.sk"),
+ &sketch.serialize(),
+ )?;
+ }
+ }
+
+ let mut union = HllUnion::new(12).unwrap();
+ for range in [0..1_000_i64, 500..520, 2_000..7_000] {
+ let mut sketch = HllSketch::new(12, HllType::Hll8).unwrap();
+ for i in range {
+ sketch.update(i);
+ }
+ union.update(&sketch);
+ }
+ for (hll_type, prefix) in TYPES {
+ write_snapshot(
+ output_dir,
+ &format!("{prefix}_union_rust.sk"),
+ &union.serialize(hll_type),
+ )?;
+ }
+
+ Ok(())
+}
diff --git a/datagen/src/kll.rs b/datagen/src/kll.rs
new file mode 100644
index 0000000..1c17bb0
--- /dev/null
+++ b/datagen/src/kll.rs
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+use std::path::Path;
+
+use datasketches::kll::KllFloat;
+use datasketches::kll::KllSketch;
+
+use crate::N_STD;
+use crate::write_snapshot;
+
+pub fn generate(output_dir: &Path) -> io::Result<()> {
+ for n in N_STD {
+ let mut sketch = KllSketch::<KllFloat<f32>>::new(200).unwrap();
+ for i in 1..=n {
+ sketch.update(KllFloat::<f32>::new(i as f32).unwrap());
+ }
+ write_snapshot(
+ output_dir,
+ &format!("kll_float_n{n}_rust.sk"),
+ &sketch.serialize(),
+ )?;
+
+ let mut sketch = KllSketch::<KllFloat<f64>>::new(200).unwrap();
+ for i in 1..=n {
+ sketch.update(KllFloat::<f64>::new(i as f64).unwrap());
+ }
+ write_snapshot(
+ output_dir,
+ &format!("kll_double_n{n}_rust.sk"),
+ &sketch.serialize(),
+ )?;
+
+ let mut sketch = KllSketch::<i64>::new(200).unwrap();
+ for i in 1..=n {
+ sketch.update(i as i64);
+ }
+ write_snapshot(
+ output_dir,
+ &format!("kll_long_n{n}_rust.sk"),
+ &sketch.serialize(),
+ )?;
+
+ let mut sketch = KllSketch::<String>::new(200).unwrap();
+ let width = n.to_string().len();
+ for i in 1..=n {
+ sketch.update(format!("{i:>width$}"));
+ }
+ write_snapshot(
+ output_dir,
+ &format!("kll_string_n{n}_rust.sk"),
+ &sketch.serialize(),
+ )?;
+ }
+ Ok(())
+}
diff --git a/datagen/src/main.rs b/datagen/src/main.rs
new file mode 100644
index 0000000..41c1d56
--- /dev/null
+++ b/datagen/src/main.rs
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fs;
+use std::io;
+use std::path::Path;
+use std::path::PathBuf;
+
+use clap::Parser;
+use clap::ValueEnum;
+
+mod bloom;
+mod countmin;
+mod cpc;
+mod density;
+mod frequencies;
+mod hll;
+mod kll;
+mod req;
+mod tdigest;
+mod theta;
+mod tuple;
+mod xor;
+
+const N_STD: [u64; 8] = [0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000];
+
+#[derive(Parser)]
+#[command(about = "Generate Rust DataSketches serialization compatibility
snapshots.")]
+struct Args {
+ #[arg(
+ long,
+ value_name = "DIR",
+ default_value = "serialization_test_data/rust_generated_files",
+ help = "Directory to write generated .sk snapshot files into."
+ )]
+ output_dir: PathBuf,
+
+ #[arg(
+ long = "sketch",
+ value_name = "FAMILY",
+ help = "Sketch family to generate (repeatable; all families when
omitted)."
+ )]
+ sketches: Vec<SketchFamily>,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)]
+enum SketchFamily {
+ Bloom,
+ #[value(name = "countmin")]
+ CountMin,
+ Cpc,
+ Density,
+ Frequencies,
+ Hll,
+ Kll,
+ Req,
+ #[value(name = "tdigest")]
+ TDigest,
+ Theta,
+ Tuple,
+ Xor,
+}
+
+impl SketchFamily {
+ fn generate(self, output_dir: &Path) -> io::Result<()> {
+ match self {
+ Self::Bloom => bloom::generate(output_dir),
+ Self::CountMin => countmin::generate(output_dir),
+ Self::Cpc => cpc::generate(output_dir),
+ Self::Density => density::generate(output_dir),
+ Self::Frequencies => frequencies::generate(output_dir),
+ Self::Hll => hll::generate(output_dir),
+ Self::Kll => kll::generate(output_dir),
+ Self::Req => req::generate(output_dir),
+ Self::TDigest => tdigest::generate(output_dir),
+ Self::Theta => theta::generate(output_dir),
+ Self::Tuple => tuple::generate(output_dir),
+ Self::Xor => xor::generate(output_dir),
+ }
+ }
+}
+
+fn write_snapshot(output_dir: &Path, name: &str, bytes: &[u8]) ->
io::Result<()> {
+ fs::write(output_dir.join(name), bytes)
+}
+
+fn main() {
+ let args = Args::parse();
+ if let Err(error) = run(args) {
+ eprintln!("failed to generate serialization snapshots: {error}");
+ std::process::exit(1);
+ }
+}
+
+fn run(args: Args) -> io::Result<()> {
+ fs::create_dir_all(&args.output_dir)?;
+ for &family in SketchFamily::value_variants() {
+ if args.sketches.is_empty() || args.sketches.contains(&family) {
+ family.generate(&args.output_dir)?;
+ }
+ }
+ Ok(())
+}
diff --git a/datagen/src/req.rs b/datagen/src/req.rs
new file mode 100644
index 0000000..3eae396
--- /dev/null
+++ b/datagen/src/req.rs
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+use std::path::Path;
+
+use datasketches::req::RankAccuracy;
+use datasketches::req::ReqFloat;
+use datasketches::req::ReqSketch;
+
+use crate::N_STD;
+use crate::write_snapshot;
+
+pub fn generate(output_dir: &Path) -> io::Result<()> {
+ for n in N_STD {
+ let mut sketch = ReqSketch::<ReqFloat<f32>>::new(12,
RankAccuracy::HighRank).unwrap();
+ for i in 1..=n {
+ sketch.update(ReqFloat::<f32>::new(i as f32).unwrap());
+ }
+ write_snapshot(
+ output_dir,
+ &format!("req_float_n{n}_rust.sk"),
+ &sketch.serialize(),
+ )?;
+ }
+
+ for m in [1_u64, 10] {
+ let mut sketch = ReqSketch::<ReqFloat<f32>>::new(12,
RankAccuracy::HighRank).unwrap();
+ for i in 1..=m {
+ sketch.update(ReqFloat::<f32>::new(-(i as f32)).unwrap());
+ }
+ write_snapshot(
+ output_dir,
+ &format!("req_float_negative_n{m}_rust.sk"),
+ &sketch.serialize(),
+ )?;
+ }
+
+ for m in [1_i32, 10] {
+ let mut sketch = ReqSketch::<ReqFloat<f32>>::new(12,
RankAccuracy::HighRank).unwrap();
+ for i in -m..=m {
+ sketch.update(ReqFloat::<f32>::new(i as f32).unwrap());
+ }
+ write_snapshot(
+ output_dir,
+ &format!("req_float_mixed_n{m}_rust.sk"),
+ &sketch.serialize(),
+ )?;
+ }
+
+ Ok(())
+}
diff --git a/datagen/src/tdigest.rs b/datagen/src/tdigest.rs
new file mode 100644
index 0000000..9ebf2da
--- /dev/null
+++ b/datagen/src/tdigest.rs
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+use std::path::Path;
+
+use datasketches::tdigest::TDigestMut;
+
+use crate::N_STD;
+use crate::write_snapshot;
+
+pub fn generate(output_dir: &Path) -> io::Result<()> {
+ for n in N_STD {
+ let mut sketch = TDigestMut::new(100).unwrap();
+ for i in 1..=n {
+ sketch.update(i as f64);
+ }
+ write_snapshot(
+ output_dir,
+ &format!("tdigest_double_n{n}_rust.sk"),
+ &sketch.serialize(),
+ )?;
+ }
+ Ok(())
+}
diff --git a/datagen/src/theta.rs b/datagen/src/theta.rs
new file mode 100644
index 0000000..56e0ff4
--- /dev/null
+++ b/datagen/src/theta.rs
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+use std::path::Path;
+
+use datasketches::theta::ThetaSketchBuilder;
+
+use crate::N_STD;
+use crate::write_snapshot;
+
+pub fn generate(output_dir: &Path) -> io::Result<()> {
+ for n in N_STD {
+ let mut sketch = ThetaSketchBuilder::default().build().unwrap();
+ for i in 0..n {
+ sketch.update(i as i64);
+ }
+ write_snapshot(
+ output_dir,
+ &format!("theta_n{n}_rust.sk"),
+ &sketch.compact(true).serialize(),
+ )?;
+ }
+
+ for n in [10_u64, 100, 1_000, 10_000, 100_000, 1_000_000] {
+ let mut sketch = ThetaSketchBuilder::default().build().unwrap();
+ for i in 0..n {
+ sketch.update(i as i64);
+ }
+ write_snapshot(
+ output_dir,
+ &format!("theta_compressed_n{n}_rust.sk"),
+ &sketch.compact(true).serialize_compressed(),
+ )?;
+ }
+
+ let mut non_empty_no_entries = ThetaSketchBuilder::default()
+ .sampling_probability(0.01)
+ .build()
+ .unwrap();
+ non_empty_no_entries.update(1_i64);
+ write_snapshot(
+ output_dir,
+ "theta_non_empty_no_entries_rust.sk",
+ &non_empty_no_entries.compact(true).serialize(),
+ )?;
+
+ Ok(())
+}
diff --git a/datagen/src/tuple.rs b/datagen/src/tuple.rs
new file mode 100644
index 0000000..bebd36c
--- /dev/null
+++ b/datagen/src/tuple.rs
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+use std::path::Path;
+
+use datasketches::tuple::DefaultUpdatePolicy;
+use datasketches::tuple::TupleSketchBuilder;
+
+use crate::N_STD;
+use crate::write_snapshot;
+
+pub fn generate(output_dir: &Path) -> io::Result<()> {
+ for n in N_STD {
+ let mut sketch =
TupleSketchBuilder::new(DefaultUpdatePolicy::<i32>::default())
+ .build()
+ .unwrap();
+ for i in 0..n {
+ sketch.update(i as i64, i as i32);
+ }
+ write_snapshot(
+ output_dir,
+ &format!("tuple_int_n{n}_rust.sk"),
+ &sketch.compact(true).serialize(),
+ )?;
+ }
+ Ok(())
+}
diff --git a/datagen/src/xor.rs b/datagen/src/xor.rs
new file mode 100644
index 0000000..08c7804
--- /dev/null
+++ b/datagen/src/xor.rs
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+use std::path::Path;
+
+use datasketches::xor::XorFilterBuilder;
+use datasketches::xor::XorFilterType;
+
+use crate::write_snapshot;
+
+const TYPES: [(XorFilterType, &str); 3] = [
+ (XorFilterType::Xor8, "xor8"),
+ (XorFilterType::Xor16, "xor16"),
+ (XorFilterType::Xor32, "xor32"),
+];
+
+pub fn generate(output_dir: &Path) -> io::Result<()> {
+ for (filter_type, prefix) in TYPES {
+ for n in [0_u64, 1, 10, 100, 1_000, 10_000, 100_000] {
+ let mut builder = XorFilterBuilder::new(filter_type).seed(9001);
+ for i in 0..n {
+ builder = builder.update(i as i64);
+ }
+ let filter = builder.build().unwrap();
+ write_snapshot(
+ output_dir,
+ &format!("{prefix}_n{n}_rust.sk"),
+ &filter.serialize(),
+ )?;
+ }
+ }
+ Ok(())
+}
diff --git a/datasketches/src/hll/serialization.rs
b/datasketches/src/hll/serialization.rs
index c982aad..7f31974 100644
--- a/datasketches/src/hll/serialization.rs
+++ b/datasketches/src/hll/serialization.rs
@@ -26,9 +26,11 @@ use crate::hll::estimator::Estimator;
pub const SERIAL_VERSION: u8 = 1;
pub const EMPTY_FLAG_MASK: u8 = 4;
-/// Flag indicating compact coupon or HLL4 auxiliary storage.
+/// Flag indicating compact serialization format.
///
-/// HLL register arrays have the same layout in compact and updatable images.
+/// Coupon lists, coupon hash sets, and HLL4 auxiliary exceptions omit empty
slots in compact
+/// images; HLL6 and HLL8 register arrays have the same layout in both forms,
and serialized
+/// sketches set this flag consistently across all modes and target types.
pub const COMPACT_FLAG_MASK: u8 = 8;
/// Flag indicating that HIP history is unavailable and composite estimation
is required.
pub const OUT_OF_ORDER_FLAG_MASK: u8 = 16;
@@ -80,10 +82,10 @@ pub fn write_hll_preamble(
aux_count: u32,
estimator: &Estimator,
) {
- let (tgt_type, compact) = match hll_type {
- HllType::Hll4 => (TGT_HLL4, true),
- HllType::Hll6 => (TGT_HLL6, false),
- HllType::Hll8 => (TGT_HLL8, false),
+ let tgt_type = match hll_type {
+ HllType::Hll4 => TGT_HLL4,
+ HllType::Hll6 => TGT_HLL6,
+ HllType::Hll8 => TGT_HLL8,
};
bytes.write_u8(HLL_PREINTS);
@@ -92,7 +94,7 @@ pub fn write_hll_preamble(
bytes.write_u8(lg_config_k);
bytes.write_u8(0); // unused in HLL6/HLL8 and compact HLL4
- let mut flags = if compact { COMPACT_FLAG_MASK } else { 0 };
+ let mut flags = COMPACT_FLAG_MASK;
if estimator.uses_composite_estimate() {
flags |= OUT_OF_ORDER_FLAG_MASK;
}
diff --git a/tests-integration/tests/serde_tests/.gitignore
b/tests-integration/tests/serde_tests/.gitignore
index d642f7a..1cc0abf 100644
--- a/tests-integration/tests/serde_tests/.gitignore
+++ b/tests-integration/tests/serde_tests/.gitignore
@@ -1,3 +1,4 @@
cpp_generated_files
go_generated_files
java_generated_files
+rust_generated_files
diff --git a/tests-integration/tests/serde_tests/bloom.rs
b/tests-integration/tests/serde_tests/bloom.rs
index 68e26f3..3120b8d 100644
--- a/tests-integration/tests/serde_tests/bloom.rs
+++ b/tests-integration/tests/serde_tests/bloom.rs
@@ -21,6 +21,7 @@ use std::path::PathBuf;
use datasketches::bloom::BloomFilter;
use datasketches::bloom::BloomFilterBuilder;
use datasketches::error::ErrorKind;
+use datasketches::hash::value::canonical_float;
use googletest::assert_that;
use googletest::prelude::gt;
@@ -131,6 +132,45 @@ fn test_go_compatibility() {
}
}
+#[test]
+fn test_rust_compatibility() {
+ for (n, num_hashes) in [
+ (0_u64, 3),
+ (0, 5),
+ (10_000, 3),
+ (10_000, 5),
+ (2_000_000, 3),
+ (2_000_000, 5),
+ (30_000_000, 3),
+ (30_000_000, 5),
+ ] {
+ let filename = format!("bf_n{n}_h{num_hashes}_rust.sk");
+ let path = serialization_test_data("rust_generated_files", &filename);
+ let bytes = fs::read(&path).unwrap();
+ let filter = BloomFilter::deserialize(&bytes).unwrap();
+ assert_eq!(
+ filter.capacity(),
+ n.max(1000).next_multiple_of(64),
+ "Wrong capacity in {}",
+ path.display()
+ );
+ if n > 0 {
+ assert!(
+ filter.contains(canonical_float::from_f64(f64::NAN)),
+ "Missing canonical NaN in {}",
+ path.display()
+ );
+ }
+ assert_eq!(
+ filter.serialize(),
+ bytes,
+ "Round-trip bytes differ for {}",
+ path.display()
+ );
+ test_bloom_filter_file(path, n, num_hashes);
+ }
+}
+
#[test]
fn test_cached_num_bits_set_is_validated_or_recomputed() {
const NUM_BITS_SET_OFFSET: usize = 24;
diff --git a/tests-integration/tests/serde_tests/countmin.rs
b/tests-integration/tests/serde_tests/countmin.rs
index d2eec69..512e5f0 100644
--- a/tests-integration/tests/serde_tests/countmin.rs
+++ b/tests-integration/tests/serde_tests/countmin.rs
@@ -334,3 +334,72 @@ fn zero_seed_hash_uses_the_callers_error_kind() {
CountMinSketch::<u64>::deserialize_with_seed(&bytes,
ZERO_HASH_SEED).unwrap_err();
assert_eq!(deserialization_error.kind(), ErrorKind::InvalidData);
}
+
+fn assert_rust_v2_snapshot(filename: &str, seed: u64, n: u64) {
+ const DEFAULT_SEED: u64 = 9001;
+
+ let path = serialization_test_data("rust_generated_files", filename);
+ let bytes = fs::read(&path).unwrap();
+ assert_eq!(bytes[1], 2, "expected v2 serial version in {filename}");
+
+ let sketch = if seed == DEFAULT_SEED {
+ CountMinSketch::<i64>::deserialize(&bytes).unwrap()
+ } else {
+ assert_eq!(
+ CountMinSketch::<i64>::deserialize(&bytes)
+ .unwrap_err()
+ .kind(),
+ ErrorKind::InvalidData
+ );
+ CountMinSketch::<i64>::deserialize_with_seed(&bytes, seed).unwrap()
+ };
+
+ let expected_total_weight = (n * (n + 3) / 2) as i64;
+ assert_eq!(sketch.num_hashes(), 3, "{filename}");
+ assert_eq!(sketch.num_buckets(), 1000, "{filename}");
+ assert_eq!(sketch.seed(), seed, "{filename}");
+ assert_eq!(sketch.total_weight(), expected_total_weight, "{filename}");
+ assert_eq!(sketch.is_empty(), n == 0, "{filename}");
+
+ let mut expected = CountMinSketch::<i64>::with_seed(3, 1000,
seed).unwrap();
+ for i in 0..n {
+ expected.update_with_weight(i as i64, (i + 1) as i64);
+ expected.update_with_weight(raw_bytes::from_str(&i.to_string()), 1);
+ }
+
+ if n == 0 {
+ assert_eq!(sketch.estimate(0_i64), 0, "{filename}");
+ assert_eq!(sketch.estimate(raw_bytes::from_str("0")), 0, "{filename}");
+ } else {
+ for i in (0..n.min(200)).chain(std::iter::once(n - 1)) {
+ let key_str = i.to_string();
+ let int_est = sketch.estimate(i as i64);
+ assert!(int_est >= (i + 1) as i64, "{filename}: item {i}");
+ assert_eq!(int_est, expected.estimate(i as i64), "{filename}: item
{i}");
+
+ let str_est = sketch.estimate(raw_bytes::from_str(&key_str));
+ assert!(str_est >= 1, "{filename}: str {key_str}");
+ assert_eq!(
+ str_est,
+ expected.estimate(raw_bytes::from_str(&key_str)),
+ "{filename}: str {key_str}"
+ );
+ }
+ }
+
+ assert_eq!(bytes, expected.serialize(), "snapshot mismatch: {filename}");
+ assert_eq!(
+ sketch.serialize(),
+ bytes,
+ "round-trip bytes differ for {filename}"
+ );
+}
+
+#[test]
+fn test_deserialize_rust_v2_snapshots() {
+ for n in [0, 1, 10, 100, 1000, 10_000] {
+ let filename = format!("count_min_v2_n{n}_rust.sk");
+ assert_rust_v2_snapshot(&filename, 9001, n);
+ }
+ assert_rust_v2_snapshot("count_min_v2_seed42_n1000_rust.sk", 42, 1000);
+}
diff --git a/tests-integration/tests/serde_tests/cpc.rs
b/tests-integration/tests/serde_tests/cpc.rs
index 7df5df1..88de077 100644
--- a/tests-integration/tests/serde_tests/cpc.rs
+++ b/tests-integration/tests/serde_tests/cpc.rs
@@ -20,6 +20,8 @@ use std::path::Path;
use datasketches::cpc::CpcSketch;
use datasketches::cpc::CpcUnion;
+use datasketches::cpc::CpcWrapper;
+use datasketches::hash::value::sign_extend;
use googletest::assert_that;
use googletest::prelude::near;
@@ -164,3 +166,35 @@ fn test_go_compatibility() {
test_sketch_replay(&path, sketch, 0..n);
}
}
+
+#[test]
+fn test_rust_compatibility() {
+ for n in [0, 100, 200, 2000, 20000] {
+ let filename = format!("cpc_n{n}_rust.sk");
+ let path = serialization_test_data("rust_generated_files", &filename);
+ let sketch = test_sketch_file(&path, n);
+ let wrapper = CpcWrapper::new(&fs::read(&path).unwrap()).unwrap();
+ assert_eq!(wrapper.lg_k(), sketch.lg_k());
+ assert_eq!(wrapper.is_empty(), sketch.is_empty());
+ assert_eq!(wrapper.estimate(), sketch.estimate());
+ test_sketch_replay(&path, sketch, 0..n);
+ }
+
+ let neg_path = serialization_test_data("rust_generated_files",
"cpc_negative_one_rust.sk");
+ let neg_bytes = fs::read(&neg_path).unwrap();
+ let mut neg_sketch = CpcSketch::deserialize(&neg_bytes).unwrap();
+ assert_eq!(neg_sketch.lg_k(), 11);
+ assert!(!neg_sketch.is_empty());
+ let initial_estimate = neg_sketch.estimate();
+ assert_that!(initial_estimate, near(1.0, 0.02));
+ assert_eq!(neg_sketch.serialize(), neg_bytes);
+
+ let wrapper = CpcWrapper::new(&neg_bytes).unwrap();
+ assert_eq!(wrapper.estimate(), initial_estimate);
+
+ neg_sketch.update(sign_extend::from_i8(-1));
+ neg_sketch.update(sign_extend::from_i16(-1));
+ neg_sketch.update(sign_extend::from_i32(-1));
+ neg_sketch.update(-1_i64);
+ assert_eq!(neg_sketch.estimate(), initial_estimate);
+}
diff --git a/tests-integration/tests/serde_tests/density.rs
b/tests-integration/tests/serde_tests/density.rs
index b5fc3ea..f5b5958 100644
--- a/tests-integration/tests/serde_tests/density.rs
+++ b/tests-integration/tests/serde_tests/density.rs
@@ -15,10 +15,14 @@
// specific language governing permissions and limitations
// under the License.
+use std::fs;
+
use datasketches::density::DensitySketch;
use datasketches::density::Kernel;
use datasketches::error::ErrorKind;
+use crate::serialization_test_data;
+
// Generated from datasketches-cpp commit
70b86faa63402d7f62f5a6b1b05109d5fd1f2186.
const CPP_F64_EMPTY: &str = "030113040a00000003000000";
@@ -1087,3 +1091,123 @@ fn
cpp_trailing_empty_top_level_and_round_trip_single_level_compacted_images() {
assert_eq!(sketch.serialize(), bytes);
}
}
+
+#[test]
+fn rust_double_snapshots_round_trip() {
+ for n in [0_u64, 1, 10, 100, 1000, 10_000] {
+ let filename = format!("density_double_n{n}_rust.sk");
+ let path = serialization_test_data("rust_generated_files", &filename);
+ let bytes = fs::read(&path).unwrap();
+ let sketch = DensitySketch::<f64>::deserialize(&bytes).unwrap();
+
+ assert_eq!(sketch.k(), 10, "{filename}");
+ assert_eq!(sketch.dim(), 3, "{filename}");
+ assert_eq!(sketch.total_weight(), n, "{filename}");
+ assert_eq!(sketch.is_empty(), n == 0, "{filename}");
+ assert_eq!(sketch.is_estimation_mode(), n >= 30, "{filename}");
+ let retained_weight: u64 = sketch.iter().map(|(_, weight)|
weight).sum();
+ if n < 30 {
+ assert_eq!(retained_weight, n, "{filename}");
+ } else {
+ assert!(retained_weight > 0, "{filename}");
+ }
+
+ match n {
+ 0 => {
+ assert_eq!(sketch.num_retained(), 0, "{filename}");
+ assert_eq!(sketch.estimate(&[0.0, 0.0, 0.0]).unwrap(), None);
+ assert_eq!(bytes, decode_hex(CPP_F64_EMPTY), "{filename}");
+ }
+ 10 => {
+ assert_eq!(bytes, decode_hex(CPP_F64_EXACT_N10), "{filename}");
+ }
+ _ => {}
+ }
+
+ if n > 0 {
+ assert!(sketch.num_retained() > 0, "{filename}");
+ let (first_pt, _) = sketch.iter().next().unwrap();
+ let (last_pt, _) = sketch.iter().last().unwrap();
+ for pt in [first_pt, last_pt] {
+ let est = sketch.estimate(pt).unwrap().unwrap();
+ assert!(
+ est.is_finite() && est > 0.0 && est <= 1.0,
+ "{filename}: invalid density estimate {est} at retained
point {pt:?}"
+ );
+ }
+ let last_x = (n - 1) as f64;
+ for sample in [[0.0, 0.0, 0.0], [last_x, last_x.sqrt(), -last_x]] {
+ let est = sketch.estimate(&sample).unwrap().unwrap();
+ assert!(
+ est.is_finite() && (0.0..=1.0).contains(&est),
+ "{filename}: invalid density estimate {est} at {sample:?}"
+ );
+ if n <= 10 {
+ assert!(est > 0.0, "{filename}: expected positive exact
estimate");
+ }
+ }
+ }
+
+ assert_eq!(sketch.serialize(), bytes, "roundtrip: {filename}");
+ }
+}
+
+#[test]
+fn rust_float_snapshots_round_trip() {
+ for n in [0_u64, 1, 10, 100, 1000, 10_000] {
+ let filename = format!("density_float_n{n}_rust.sk");
+ let path = serialization_test_data("rust_generated_files", &filename);
+ let bytes = fs::read(&path).unwrap();
+ let sketch = DensitySketch::<f32>::deserialize(&bytes).unwrap();
+
+ assert_eq!(sketch.k(), 10, "{filename}");
+ assert_eq!(sketch.dim(), 3, "{filename}");
+ assert_eq!(sketch.total_weight(), n, "{filename}");
+ assert_eq!(sketch.is_empty(), n == 0, "{filename}");
+ assert_eq!(sketch.is_estimation_mode(), n >= 30, "{filename}");
+ let retained_weight: u64 = sketch.iter().map(|(_, weight)|
weight).sum();
+ if n < 30 {
+ assert_eq!(retained_weight, n, "{filename}");
+ } else {
+ assert!(retained_weight > 0, "{filename}");
+ }
+
+ match n {
+ 0 => {
+ assert_eq!(sketch.num_retained(), 0, "{filename}");
+ assert_eq!(sketch.estimate(&[0.0, 0.0, 0.0]).unwrap(), None);
+ assert_eq!(bytes, decode_hex(CPP_F32_EMPTY), "{filename}");
+ }
+ 10 => {
+ assert_eq!(bytes, decode_hex(CPP_F32_EXACT_N10), "{filename}");
+ }
+ _ => {}
+ }
+
+ if n > 0 {
+ assert!(sketch.num_retained() > 0, "{filename}");
+ let (first_pt, _) = sketch.iter().next().unwrap();
+ let (last_pt, _) = sketch.iter().last().unwrap();
+ for pt in [first_pt, last_pt] {
+ let est = sketch.estimate(pt).unwrap().unwrap();
+ assert!(
+ est.is_finite() && est > 0.0 && est <= 1.0,
+ "{filename}: invalid density estimate {est} at retained
point {pt:?}"
+ );
+ }
+ let last_x = (n - 1) as f32;
+ for sample in [[0.0, 0.0, 0.0], [last_x, last_x.sqrt(), -last_x]] {
+ let est = sketch.estimate(&sample).unwrap().unwrap();
+ assert!(
+ est.is_finite() && (0.0..=1.0).contains(&est),
+ "{filename}: invalid density estimate {est} at {sample:?}"
+ );
+ if n <= 10 {
+ assert!(est > 0.0, "{filename}: expected positive exact
estimate");
+ }
+ }
+ }
+
+ assert_eq!(sketch.serialize(), bytes, "roundtrip: {filename}");
+ }
+}
diff --git a/tests-integration/tests/serde_tests/frequencies.rs
b/tests-integration/tests/serde_tests/frequencies.rs
index e9370ea..7617865 100644
--- a/tests-integration/tests/serde_tests/frequencies.rs
+++ b/tests-integration/tests/serde_tests/frequencies.rs
@@ -21,6 +21,7 @@ use datasketches::codec::SketchBytes;
use datasketches::codec::SketchSlice;
use datasketches::error::Error;
use datasketches::error::ErrorKind;
+use datasketches::frequencies::ErrorType;
use datasketches::frequencies::FrequentItemValue;
use datasketches::frequencies::FrequentItemsSketch;
use googletest::assert_that;
@@ -189,6 +190,16 @@ fn test_java_frequent_strings_purged() {
check_purged_compatibility::<String>("java_generated_files",
"frequent_string_purged_java.sk");
}
+#[test]
+fn test_rust_frequent_longs_purged() {
+ check_purged_compatibility::<i64>("rust_generated_files",
"frequent_long_purged_rust.sk");
+}
+
+#[test]
+fn test_rust_frequent_strings_purged() {
+ check_purged_compatibility::<String>("rust_generated_files",
"frequent_string_purged_rust.sk");
+}
+
#[test]
fn test_deserialize_rejects_zero_stream_weight() {
let mut active_sketch = FrequentItemsSketch::<i64>::new(32).unwrap();
@@ -506,6 +517,145 @@ fn test_go_frequent_strings_utf8() {
assert_eq!(sketch.estimate(&"эюя".to_string()), 7);
}
+fn assert_frequent_round_trip<T>(sketch: &FrequentItemsSketch<T>, bytes: &[u8])
+where
+ T: FrequentItemValue + Clone + Ord + std::fmt::Debug,
+{
+ if sketch.num_retained() <= 1 {
+ assert_eq!(sketch.serialize(), bytes);
+ }
+
+ let reserialized = sketch.serialize();
+ let restored =
FrequentItemsSketch::<T>::deserialize(&reserialized).unwrap();
+ assert_eq!(restored.is_empty(), sketch.is_empty());
+ assert_eq!(restored.lg_max_map_size(), sketch.lg_max_map_size());
+ assert_eq!(restored.lg_cur_map_size(), sketch.lg_cur_map_size());
+ assert_eq!(restored.total_weight(), sketch.total_weight());
+ assert_eq!(restored.maximum_error(), sketch.maximum_error());
+ assert_eq!(restored.num_retained(), sketch.num_retained());
+
+ let mut expected_rows: Vec<_> = sketch
+ .frequent_items(ErrorType::NoFalseNegatives)
+ .into_iter()
+ .map(|row| {
+ (
+ row.item().clone(),
+ row.estimate(),
+ row.lower_bound(),
+ row.upper_bound(),
+ )
+ })
+ .collect();
+ expected_rows.sort_by(|left, right| left.0.cmp(&right.0));
+
+ let mut actual_rows: Vec<_> = restored
+ .frequent_items(ErrorType::NoFalseNegatives)
+ .into_iter()
+ .map(|row| {
+ (
+ row.item().clone(),
+ row.estimate(),
+ row.lower_bound(),
+ row.upper_bound(),
+ )
+ })
+ .collect();
+ actual_rows.sort_by(|left, right| left.0.cmp(&right.0));
+
+ assert_eq!(actual_rows, expected_rows);
+}
+
+#[test]
+fn test_rust_frequent_longs_compatibility() {
+ for n in [0, 1, 10, 100, 1000, 10000, 100000, 1000000] {
+ let filename = format!("frequent_long_n{n}_rust.sk");
+ let path = serialization_test_data("rust_generated_files", &filename);
+ let bytes = fs::read(&path).unwrap();
+ let sketch = FrequentItemsSketch::<i64>::deserialize(&bytes).unwrap();
+ assert_eq!(sketch.is_empty(), n == 0);
+ if n > 10 {
+ assert_that!(sketch.maximum_error(), gt(0));
+ } else {
+ assert_eq!(sketch.maximum_error(), 0);
+ for item in 1..=(n as i64) {
+ assert_eq!(sketch.estimate(&item), 1, "{filename}: item
{item}");
+ }
+ }
+ assert_eq!(sketch.total_weight(), n);
+ assert_frequent_round_trip(&sketch, &bytes);
+ }
+}
+
+#[test]
+fn test_rust_frequent_strings_compatibility() {
+ for n in [0, 1, 10, 100, 1000, 10000, 100000, 1000000] {
+ let filename = format!("frequent_string_n{n}_rust.sk");
+ let path = serialization_test_data("rust_generated_files", &filename);
+ let bytes = fs::read(&path).unwrap();
+ let sketch =
FrequentItemsSketch::<String>::deserialize(&bytes).unwrap();
+ assert_eq!(sketch.is_empty(), n == 0);
+ if n > 10 {
+ assert_that!(sketch.maximum_error(), gt(0));
+ } else {
+ assert_eq!(sketch.maximum_error(), 0);
+ for item in 1..=n {
+ assert_eq!(
+ sketch.estimate(&item.to_string()),
+ 1,
+ "{filename}: item {item}"
+ );
+ }
+ }
+ assert_eq!(sketch.total_weight(), n);
+ assert_frequent_round_trip(&sketch, &bytes);
+ }
+}
+
+#[test]
+fn test_rust_frequent_strings_ascii() {
+ let path = serialization_test_data("rust_generated_files",
"frequent_string_ascii_rust.sk");
+ let bytes = fs::read(&path).unwrap();
+ let sketch = FrequentItemsSketch::<String>::deserialize(&bytes).unwrap();
+ assert!(!sketch.is_empty());
+ assert_eq!(sketch.maximum_error(), 0);
+ assert_eq!(sketch.total_weight(), 10);
+ assert_eq!(
+ sketch.estimate(&"aaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string()),
+ 1
+ );
+ assert_eq!(
+ sketch.estimate(&"bbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string()),
+ 2
+ );
+ assert_eq!(
+ sketch.estimate(&"ccccccccccccccccccccccccccccc".to_string()),
+ 3
+ );
+ assert_eq!(
+ sketch.estimate(&"ddddddddddddddddddddddddddddd".to_string()),
+ 4
+ );
+ assert_frequent_round_trip(&sketch, &bytes);
+}
+
+#[test]
+fn test_rust_frequent_strings_utf8() {
+ let path = serialization_test_data("rust_generated_files",
"frequent_string_utf8_rust.sk");
+ let bytes = fs::read(&path).unwrap();
+ let sketch = FrequentItemsSketch::<String>::deserialize(&bytes).unwrap();
+ assert!(!sketch.is_empty());
+ assert_eq!(sketch.maximum_error(), 0);
+ assert_eq!(sketch.total_weight(), 28);
+ assert_eq!(sketch.estimate(&"абвгд".to_string()), 1);
+ assert_eq!(sketch.estimate(&"еёжзи".to_string()), 2);
+ assert_eq!(sketch.estimate(&"йклмн".to_string()), 3);
+ assert_eq!(sketch.estimate(&"опрст".to_string()), 4);
+ assert_eq!(sketch.estimate(&"уфхцч".to_string()), 5);
+ assert_eq!(sketch.estimate(&"шщъыь".to_string()), 6);
+ assert_eq!(sketch.estimate(&"эюя".to_string()), 7);
+ assert_frequent_round_trip(&sketch, &bytes);
+}
+
#[test]
fn test_deserialize_rejects_invalid_map_sizes() {
let empty = FrequentItemsSketch::<i64>::new(32).unwrap().serialize();
diff --git a/tests-integration/tests/serde_tests/hll.rs
b/tests-integration/tests/serde_tests/hll.rs
index 9e30981..b0b0a8b 100644
--- a/tests-integration/tests/serde_tests/hll.rs
+++ b/tests-integration/tests/serde_tests/hll.rs
@@ -605,6 +605,86 @@ fn test_go_hll8_compatibility() {
}
}
+fn assert_rust_hll_compatibility(type_name: &str, expected_type: HllType) {
+ for n in [0, 1, 10, 100, 1000, 10000, 100000, 1000000] {
+ let rust_filename = format!("{type_name}_n{n}_rust.sk");
+ let rust_path = serialization_test_data("rust_generated_files",
&rust_filename);
+ let java_filename = format!("{type_name}_n{n}_java.sk");
+ let java_path = serialization_test_data("java_generated_files",
&java_filename);
+
+ let rust_bytes = fs::read(&rust_path).unwrap();
+ let java_bytes = fs::read(&java_path).unwrap();
+ assert_ne!(rust_bytes[5] & 0x08, 0, "{rust_filename}");
+ assert_eq!(rust_bytes.len(), java_bytes.len(), "{rust_filename}");
+ if expected_type == HllType::Hll4 && n >= 100_000 {
+ // Compact HLL4 stores 0 for lg_arr (byte 4), while Java records
its internal aux
+ // table lg_arr; preamble, nibbles, and compact aux coupons
otherwise match bit-for-bit.
+ assert_eq!(rust_bytes[..4], java_bytes[..4], "{rust_filename}");
+ assert_eq!(rust_bytes[4], 0, "{rust_filename}");
+ assert_eq!(rust_bytes[5..], java_bytes[5..], "{rust_filename}");
+ } else {
+ assert_eq!(
+ rust_bytes, java_bytes,
+ "Rust {rust_filename} bytes differ from {java_filename}"
+ );
+ }
+
+ let sketch = HllSketch::deserialize(&rust_bytes).unwrap();
+ let java_sketch = HllSketch::deserialize(&java_bytes).unwrap();
+ assert_eq!(sketch.target_type(), expected_type, "{rust_filename}");
+ assert_eq!(sketch.is_empty(), n == 0, "{rust_filename}");
+ if n <= 1 || (expected_type != HllType::Hll4 && n >= 1000) {
+ assert_eq!(sketch.serialize(), rust_bytes, "{rust_filename}");
+ }
+ assert_hll_sketches_eq(&sketch, &java_sketch, &rust_filename);
+ test_sketch_file(rust_path, n, 12);
+ }
+}
+
+#[test]
+fn test_rust_hll4_compatibility() {
+ assert_rust_hll_compatibility("hll4", HllType::Hll4);
+}
+
+#[test]
+fn test_rust_hll6_compatibility() {
+ assert_rust_hll_compatibility("hll6", HllType::Hll6);
+}
+
+#[test]
+fn test_rust_hll8_compatibility() {
+ assert_rust_hll_compatibility("hll8", HllType::Hll8);
+}
+
+#[test]
+fn test_rust_hll_union_compatibility() {
+ for (type_name, expected_type) in [
+ ("hll4", HllType::Hll4),
+ ("hll6", HllType::Hll6),
+ ("hll8", HllType::Hll8),
+ ] {
+ let filename = format!("{type_name}_union_rust.sk");
+ let path = serialization_test_data("rust_generated_files", &filename);
+ let bytes = fs::read(&path).unwrap();
+ assert_ne!(bytes[5] & 0x08, 0, "{filename}");
+ let sketch = HllSketch::deserialize(&bytes).unwrap();
+
+ assert_eq!(sketch.lg_config_k(), 12, "{filename}");
+ assert_eq!(sketch.target_type(), expected_type, "{filename}");
+ assert!(!sketch.is_empty(), "{filename}");
+ assert_that!(
+ 6000.0,
+ all!(
+ ge(sketch.lower_bound(NumStdDev::Two)),
+ le(sketch.upper_bound(NumStdDev::Two))
+ ),
+ "{filename}"
+ );
+
+ test_sketch_file(path, 6000, 12);
+ }
+}
+
#[test]
fn test_estimate_accuracy() {
// This test verifies and prints actual estimates to show accuracy
diff --git a/tests-integration/tests/serde_tests/kll.rs
b/tests-integration/tests/serde_tests/kll.rs
index d9db311..de107f6 100644
--- a/tests-integration/tests/serde_tests/kll.rs
+++ b/tests-integration/tests/serde_tests/kll.rs
@@ -396,6 +396,86 @@ fn test_go_kll_string_compatibility() {
}
}
+#[test]
+fn test_rust_kll_float_compatibility() {
+ for n in [0, 1, 10, 100, 1000, 10000, 100000, 1000000] {
+ let path =
+ serialization_test_data("rust_generated_files",
&format!("kll_float_n{n}_rust.sk"));
+ test_f32_file(path, n);
+ }
+}
+
+#[test]
+fn test_rust_kll_double_compatibility() {
+ for n in [0, 1, 10, 100, 1000, 10000, 100000, 1000000] {
+ let path =
+ serialization_test_data("rust_generated_files",
&format!("kll_double_n{n}_rust.sk"));
+ test_f64_file(path, n);
+ }
+}
+
+#[test]
+fn test_rust_kll_long_compatibility() {
+ for n in [0, 1, 10, 100, 1000, 10000, 100000, 1000000] {
+ let path =
+ serialization_test_data("rust_generated_files",
&format!("kll_long_n{n}_rust.sk"));
+ test_i64_file(path, n);
+ }
+}
+
+#[test]
+fn test_rust_kll_string_compatibility() {
+ for n in [0, 1, 10, 100, 1000, 10000, 100000, 1000000] {
+ let filename = format!("kll_string_n{n}_rust.sk");
+ let path = serialization_test_data("rust_generated_files", &filename);
+ let bytes = fs::read(&path).unwrap();
+ let sketch = KllSketch::<String>::deserialize(&bytes)
+ .unwrap_or_else(|error| panic!("{}: {error}", path.display()));
+
+ assert_eq!(sketch.k(), DEFAULT_K, "wrong k in {}", path.display());
+ assert_eq!(
+ sketch.total_weight() as usize,
+ n,
+ "wrong n in {}",
+ path.display()
+ );
+ assert_eq!(
+ sketch.is_estimation_mode(),
+ n > DEFAULT_K as usize,
+ "wrong estimation mode in {}",
+ path.display()
+ );
+ assert_eq!(
+ sketch.is_empty(),
+ n == 0,
+ "wrong empty flag in {}",
+ path.display()
+ );
+
+ if n == 0 {
+ assert!(sketch.min_item().is_none(), "min should be None");
+ assert!(sketch.max_item().is_none(), "max should be None");
+ } else {
+ let width = n.to_string().len();
+ assert_eq!(
+ sketch.min_item(),
+ Some(&format!("{:>width$}", 1)),
+ "min item mismatch in {}",
+ path.display()
+ );
+ assert_eq!(
+ sketch.max_item(),
+ Some(&n.to_string()),
+ "max item mismatch in {}",
+ path.display()
+ );
+ }
+ assert_eq!(sketch.serialize(), bytes, "roundtrip: {}", path.display());
+
+ test_string_file(path, n);
+ }
+}
+
#[test]
fn test_custom_kll_value_roundtrip() {
let mut sketch = KllSketch::<Record>::new(64).unwrap();
diff --git a/tests-integration/tests/serde_tests/req.rs
b/tests-integration/tests/serde_tests/req.rs
index 452477d..0ff5577 100644
--- a/tests-integration/tests/serde_tests/req.rs
+++ b/tests-integration/tests/serde_tests/req.rs
@@ -561,3 +561,105 @@ fn java_compatibility() {
validate_cross_language_fixture(path, n);
}
}
+
+#[test]
+fn rust_compatibility() {
+ for n in [0u64, 1, 10, 100, 1000, 10000, 100000, 1000000] {
+ let path =
+ serialization_test_data("rust_generated_files",
&format!("req_float_n{n}_rust.sk"));
+ validate_cross_language_fixture(path, n);
+ }
+
+ for m in [1u64, 10] {
+ let neg_name = format!("req_float_negative_n{m}_rust.sk");
+ let neg_path = serialization_test_data("rust_generated_files",
&neg_name);
+ let neg_bytes = fs::read(&neg_path).unwrap();
+ let neg = ReqSketch::<ReqF32>::deserialize(&neg_bytes).unwrap();
+ let neg_min = req_f32(-(m as f32));
+ let neg_max = req_f32(-1.0);
+ assert_eq!(neg.k(), 12, "{neg_name}");
+ assert_eq!(neg.rank_accuracy(), RankAccuracy::HighRank, "{neg_name}");
+ assert!(!neg.is_empty(), "{neg_name}");
+ assert!(!neg.is_estimation_mode(), "{neg_name}");
+ assert_eq!(neg.total_weight(), m, "{neg_name}");
+ assert_eq!(neg.min_item(), Some(&neg_min), "{neg_name}");
+ assert_eq!(neg.max_item(), Some(&neg_max), "{neg_name}");
+ assert_eq!(
+ neg.quantile(0.0, SearchCriteria::Inclusive).unwrap(),
+ Some(neg_min),
+ "{neg_name}"
+ );
+ assert_eq!(
+ neg.quantile(1.0, SearchCriteria::Inclusive).unwrap(),
+ Some(neg_max),
+ "{neg_name}"
+ );
+ assert_eq!(
+ neg.rank(&neg_min, SearchCriteria::Exclusive),
+ Some(0.0),
+ "{neg_name}"
+ );
+ assert_eq!(
+ neg.rank(&neg_min, SearchCriteria::Inclusive),
+ Some(1.0 / m as f64),
+ "{neg_name}"
+ );
+ assert_eq!(
+ neg.rank(&neg_max, SearchCriteria::Inclusive),
+ Some(1.0),
+ "{neg_name}"
+ );
+ assert_eq!(neg.serialize(), neg_bytes, "{neg_name}");
+
+ let mixed_name = format!("req_float_mixed_n{m}_rust.sk");
+ let mixed_path = serialization_test_data("rust_generated_files",
&mixed_name);
+ let mixed_bytes = fs::read(&mixed_path).unwrap();
+ let mixed = ReqSketch::<ReqF32>::deserialize(&mixed_bytes).unwrap();
+ let expected_n = 2 * m + 1;
+ let mixed_min = req_f32(-(m as f32));
+ let zero = req_f32(0.0);
+ let mixed_max = req_f32(m as f32);
+ assert_eq!(mixed.k(), 12, "{mixed_name}");
+ assert_eq!(
+ mixed.rank_accuracy(),
+ RankAccuracy::HighRank,
+ "{mixed_name}"
+ );
+ assert!(!mixed.is_empty(), "{mixed_name}");
+ assert!(!mixed.is_estimation_mode(), "{mixed_name}");
+ assert_eq!(mixed.total_weight(), expected_n, "{mixed_name}");
+ assert_eq!(mixed.min_item(), Some(&mixed_min), "{mixed_name}");
+ assert_eq!(mixed.max_item(), Some(&mixed_max), "{mixed_name}");
+ assert_eq!(
+ mixed.quantile(0.0, SearchCriteria::Inclusive).unwrap(),
+ Some(mixed_min),
+ "{mixed_name}"
+ );
+ assert_eq!(
+ mixed.quantile(0.5, SearchCriteria::Inclusive).unwrap(),
+ Some(zero),
+ "{mixed_name}"
+ );
+ assert_eq!(
+ mixed.quantile(1.0, SearchCriteria::Inclusive).unwrap(),
+ Some(mixed_max),
+ "{mixed_name}"
+ );
+ assert_eq!(
+ mixed.rank(&zero, SearchCriteria::Exclusive),
+ Some(m as f64 / expected_n as f64),
+ "{mixed_name}"
+ );
+ assert_eq!(
+ mixed.rank(&zero, SearchCriteria::Inclusive),
+ Some((m + 1) as f64 / expected_n as f64),
+ "{mixed_name}"
+ );
+ assert_eq!(
+ mixed.rank(&mixed_max, SearchCriteria::Inclusive),
+ Some(1.0),
+ "{mixed_name}"
+ );
+ assert_eq!(mixed.serialize(), mixed_bytes, "{mixed_name}");
+ }
+}
diff --git a/tests-integration/tests/serde_tests/tdigest.rs
b/tests-integration/tests/serde_tests/tdigest.rs
index 1006e44..d9c7671 100644
--- a/tests-integration/tests/serde_tests/tdigest.rs
+++ b/tests-integration/tests/serde_tests/tdigest.rs
@@ -177,6 +177,15 @@ fn test_deserialize_from_go_snapshots() {
}
}
+#[test]
+fn test_deserialize_from_rust_snapshots() {
+ for n in [0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000] {
+ let filename = format!("tdigest_double_n{n}_rust.sk");
+ let path = serialization_test_data("rust_generated_files", &filename);
+ test_sketch_file(path, n, false, false);
+ }
+}
+
#[test]
fn test_empty() {
let mut td = TDigestMut::new(100).unwrap();
diff --git a/tests-integration/tests/serde_tests/theta.rs
b/tests-integration/tests/serde_tests/theta.rs
index 4073d0f..5e8df92 100644
--- a/tests-integration/tests/serde_tests/theta.rs
+++ b/tests-integration/tests/serde_tests/theta.rs
@@ -160,6 +160,70 @@ fn test_go_compatibility() {
test_sketch_file(path, 0, false);
}
+#[test]
+fn test_rust_compatibility() {
+ for n in [0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000] {
+ let rust_filename = format!("theta_n{n}_rust.sk");
+ let rust_path = serialization_test_data("rust_generated_files",
&rust_filename);
+ let java_filename = format!("theta_n{n}_java.sk");
+ let java_path = serialization_test_data("java_generated_files",
&java_filename);
+
+ let rust_bytes = fs::read(&rust_path).unwrap();
+ let java_bytes = fs::read(&java_path).unwrap();
+ if n == 1 {
+ // Java sets the optional v3 SINGLE_ITEM flag (0x20) in byte 5;
all other header and
+ // hash payload bytes match bit-for-bit.
+ assert_eq!(rust_bytes.len(), java_bytes.len(), "{rust_filename}");
+ assert_eq!(rust_bytes[..5], java_bytes[..5], "{rust_filename}");
+ assert_eq!(rust_bytes[5] | 0x20, java_bytes[5], "{rust_filename}");
+ assert_eq!(rust_bytes[6..], java_bytes[6..], "{rust_filename}");
+ } else {
+ assert_eq!(
+ rust_bytes, java_bytes,
+ "Rust {rust_filename} bytes differ from {java_filename}"
+ );
+ }
+ let sketch = CompactThetaSketch::deserialize(&rust_bytes).unwrap();
+ assert_eq!(sketch.is_empty(), n == 0, "{rust_filename}");
+ assert_eq!(sketch.serialize(), rust_bytes, "{rust_filename}");
+
+ test_sketch_file(rust_path, n, false);
+ }
+
+ for n in [10, 100, 1000, 10_000, 100_000, 1_000_000] {
+ let rust_filename = format!("theta_compressed_n{n}_rust.sk");
+ let rust_path = serialization_test_data("rust_generated_files",
&rust_filename);
+ let java_filename = format!("theta_compressed_n{n}_java.sk");
+ let java_path = serialization_test_data("java_generated_files",
&java_filename);
+
+ let rust_bytes = fs::read(&rust_path).unwrap();
+ let java_bytes = fs::read(&java_path).unwrap();
+ assert_eq!(
+ rust_bytes, java_bytes,
+ "Rust {rust_filename} bytes differ from {java_filename}"
+ );
+ let sketch = CompactThetaSketch::deserialize(&rust_bytes).unwrap();
+ assert_eq!(sketch.serialize_compressed(), rust_bytes,
"{rust_filename}");
+
+ test_sketch_file(rust_path, n, true);
+ }
+
+ let rust_path =
+ serialization_test_data("rust_generated_files",
"theta_non_empty_no_entries_rust.sk");
+ let java_path =
+ serialization_test_data("java_generated_files",
"theta_non_empty_no_entries_java.sk");
+ let rust_bytes = fs::read(&rust_path).unwrap();
+ let java_bytes = fs::read(&java_path).unwrap();
+ assert_eq!(rust_bytes, java_bytes);
+
+ let sketch = CompactThetaSketch::deserialize(&rust_bytes).unwrap();
+ assert!(!sketch.is_empty());
+ assert_eq!(sketch.num_retained(), 0);
+ assert!(sketch.is_estimation_mode());
+ assert_eq!(sketch.serialize(), rust_bytes);
+ test_sketch_file(rust_path, 0, false);
+}
+
#[test]
fn empty_sketch_serialization() {
let expected = fs::read(serialization_test_data(
diff --git a/tests-integration/tests/serde_tests/tuple.rs
b/tests-integration/tests/serde_tests/tuple.rs
index 679923c..91a69bc 100644
--- a/tests-integration/tests/serde_tests/tuple.rs
+++ b/tests-integration/tests/serde_tests/tuple.rs
@@ -120,6 +120,19 @@ fn test_go_compatibility() {
}
}
+#[test]
+fn test_rust_compatibility() {
+ for n in [0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000] {
+ let filename = format!("tuple_int_n{n}_rust.sk");
+ let path = serialization_test_data("rust_generated_files", &filename);
+ let bytes = fs::read(&path).unwrap();
+ let sketch = CompactTupleSketch::<i32>::deserialize(&bytes).unwrap();
+ assert!(sketch.is_ordered(), "{filename}");
+ assert_eq!(sketch.serialize(), bytes, "roundtrip: {filename}");
+ test_sketch_file(path, n);
+ }
+}
+
#[test]
fn empty_sketch_serialization() {
let expected = fs::read(serialization_test_data(
diff --git a/tests-integration/tests/serde_tests/xor.rs
b/tests-integration/tests/serde_tests/xor.rs
index 0c05506..8553c2b 100644
--- a/tests-integration/tests/serde_tests/xor.rs
+++ b/tests-integration/tests/serde_tests/xor.rs
@@ -15,11 +15,15 @@
// specific language governing permissions and limitations
// under the License.
+use std::fs;
+
use datasketches::error::ErrorKind;
use datasketches::xor::XorFilter;
use datasketches::xor::XorFilterBuilder;
use datasketches::xor::XorFilterType;
+use crate::serialization_test_data;
+
// Generated from datasketches-java commit
1ed3eab435a4767d672ce5797eb53f66b14d78bb.
// XorFilterBuilder.create([1, 2, 3, Long.MIN_VALUE], bits, 42).toByteArray().
const JAVA_XOR8_IMAGE: &str = concat!(
@@ -219,3 +223,47 @@ fn xor32_image_preserves_full_width_little_endian_slots() {
assert!(filter.contains_hash(0));
assert_eq!(filter.serialize(), bytes);
}
+
+#[test]
+fn rust_snapshots_round_trip() {
+ for (prefix, expected_type, max_false_positives) in [
+ ("xor8", XorFilterType::Xor8, 120usize),
+ ("xor16", XorFilterType::Xor16, 5),
+ ("xor32", XorFilterType::Xor32, 1),
+ ] {
+ for n in [0_u64, 1, 10, 100, 1000, 10_000, 100_000] {
+ let filename = format!("{prefix}_n{n}_rust.sk");
+ let path = serialization_test_data("rust_generated_files",
&filename);
+ let bytes = fs::read(&path).unwrap();
+ let filter = XorFilter::deserialize(&bytes).unwrap();
+
+ assert_eq!(filter.filter_type(), expected_type, "{filename}");
+
+ for key in 0..n.min(1000) {
+ assert!(
+ filter.contains(key as i64),
+ "{filename}: false negative on {key}"
+ );
+ }
+ if n > 1000 {
+ assert!(
+ filter.contains((n - 1) as i64),
+ "{filename}: false negative on {}",
+ n - 1
+ );
+ }
+
+ if n > 0 {
+ let false_positives = (1_000_000_i64..1_010_000_i64)
+ .filter(|&key| filter.contains(key))
+ .count();
+ assert!(
+ false_positives <= max_false_positives,
+ "{filename}: excessive false positives ({false_positives}
> {max_false_positives})"
+ );
+ }
+
+ assert_eq!(filter.serialize(), bytes, "roundtrip: {filename}");
+ }
+ }
+}
diff --git a/xtask/src/main.rs b/xtask/src/main.rs
index ff88cc4..46f72a4 100644
--- a/xtask/src/main.rs
+++ b/xtask/src/main.rs
@@ -15,19 +15,15 @@
// specific language governing permissions and limitations
// under the License.
-use std::error::Error;
-use std::fs;
-use std::io;
+mod prepare_testdata;
+
use std::path::Path;
use std::process::Command as StdCommand;
-use std::time::Duration;
use clap::Parser;
use clap::Subcommand;
-use flate2::read::GzDecoder;
-use tar::Archive;
-type Result<T> = std::result::Result<T, Box<dyn Error>>;
+use crate::prepare_testdata::CommandPrepareTestData;
fn main() {
let cmd = Command::parse();
@@ -311,117 +307,3 @@ fn make_taplo_cmd(fix: bool) -> StdCommand {
}
cmd
}
-
-#[derive(Parser)]
-#[clap(name = "prepare-testdata")]
-struct CommandPrepareTestData {
- #[arg(
- value_name = "LANG",
- value_parser = ["java", "cpp", "c++", "go", "golang"],
- help = "Languages to prepare (all by default)."
- )]
- langs: Vec<String>,
-}
-
-impl CommandPrepareTestData {
- fn run(self) {
- if let Err(error) = self.prepare() {
- eprintln!("failed to prepare serialization test data: {error}");
- std::process::exit(1);
- }
- }
-
- fn prepare(self) -> Result<()> {
- const REVISION: &str = "8193bab45ca5da3b513ea43fb5793e7e5b4da548";
- let serde_tests =
-
Path::new(env!("CARGO_WORKSPACE_DIR")).join("tests-integration/tests/serde_tests");
- let archive_url =
-
format!("https://api.github.com/repos/apache/datasketches-tck/tarball/{REVISION}");
-
- println!("Downloading serialization snapshots from {archive_url}");
-
- let timeout = Some(Duration::from_secs(60));
- let agent: ureq::Agent = ureq::Agent::config_builder()
- .timeout_connect(timeout)
- .timeout_recv_response(timeout)
- .timeout_recv_body(timeout)
- .build()
- .into();
- let response = agent
- .get(&archive_url)
- .header("Accept", "application/vnd.github+json")
- .header("X-GitHub-Api-Version", "2026-03-10")
- .call()?;
- let (_, body) = response.into_parts();
- let mut archive = Archive::new(GzDecoder::new(body.into_reader()));
-
- let mut targets = vec![];
- for language in self.languages() {
- let directory_name = format!("{language}_generated_files");
- let source_directory =
Path::new("serialization_test_data").join(&directory_name);
- let destination = serde_tests.join(directory_name);
- if fs::exists(&destination)? {
- println!(
- "Removing existing {language} snapshots from {}",
- destination.display()
- );
- fs::remove_dir_all(&destination)?;
- }
- fs::create_dir_all(&destination)?;
- targets.push((language, source_directory, destination, 0_usize));
- }
-
- for member in archive.entries()? {
- let mut member = member?;
- let path = member.path()?;
- if !member.header().entry_type().is_file()
- || path.extension().is_none_or(|extension| extension != "sk")
- {
- continue;
- }
-
- let Some((_, _, destination, count)) =
targets.iter_mut().find(|target| {
- path.parent()
- .is_some_and(|parent| parent.ends_with(&target.1))
- }) else {
- continue;
- };
-
- let name = path.file_name().expect("snapshot path has a file
name");
- let mut output = fs::File::create(destination.join(name))?;
- io::copy(&mut member, &mut output)?;
- *count += 1;
- }
-
- for (language, _, destination, count) in targets {
- if count == 0 {
- return Err(format!("no {language} snapshots found in the TCK
archive").into());
- }
- println!(
- "Extracted {count} {language} snapshots into {}",
- destination.display()
- );
- }
- Ok(())
- }
-
- fn languages(&self) -> Vec<&'static str> {
- if self.langs.is_empty() {
- return vec!["cpp", "go", "java"];
- }
-
- let mut languages = vec![];
- for language in &self.langs {
- let language = match language.as_str() {
- "java" => "java",
- "cpp" | "c++" => "cpp",
- "go" | "golang" => "go",
- _ => unreachable!("language is validated by clap"),
- };
- if !languages.contains(&language) {
- languages.push(language);
- }
- }
- languages
- }
-}
diff --git a/xtask/src/prepare_testdata.rs b/xtask/src/prepare_testdata.rs
new file mode 100644
index 0000000..87038ca
--- /dev/null
+++ b/xtask/src/prepare_testdata.rs
@@ -0,0 +1,208 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::error::Error;
+use std::fs;
+use std::io;
+use std::path::Path;
+use std::time::Duration;
+
+use clap::Parser;
+use flate2::read::GzDecoder;
+use tar::Archive;
+
+use crate::find_command;
+
+type Result<T> = std::result::Result<T, Box<dyn Error>>;
+
+#[derive(Parser)]
+#[clap(name = "prepare-testdata")]
+pub struct CommandPrepareTestData {
+ #[arg(
+ value_name = "LANG",
+ value_parser = ["java", "cpp", "c++", "go", "golang", "rust"],
+ help = "Languages to prepare (all by default)."
+ )]
+ langs: Vec<String>,
+}
+
+impl CommandPrepareTestData {
+ pub fn run(self) {
+ if let Err(error) = self.prepare() {
+ eprintln!("failed to prepare serialization test data: {error}");
+ std::process::exit(1);
+ }
+ }
+
+ fn prepare(self) -> Result<()> {
+ let serde_tests =
+
Path::new(env!("CARGO_WORKSPACE_DIR")).join("tests-integration/tests/serde_tests");
+ let languages = self.languages();
+ let remote_languages: Vec<&'static str> = languages
+ .iter()
+ .copied()
+ .filter(|&language| language != "rust")
+ .collect();
+
+ if !remote_languages.is_empty() {
+ prepare_remote(&serde_tests, &remote_languages)?;
+ }
+ if languages.contains(&"rust") {
+ prepare_rust(&serde_tests)?;
+ }
+ Ok(())
+ }
+
+ fn languages(&self) -> Vec<&'static str> {
+ if self.langs.is_empty() {
+ return vec!["cpp", "go", "java", "rust"];
+ }
+
+ let mut languages = vec![];
+ for language in &self.langs {
+ let language = match language.as_str() {
+ "java" => "java",
+ "cpp" | "c++" => "cpp",
+ "go" | "golang" => "go",
+ "rust" => "rust",
+ _ => unreachable!("language is validated by clap"),
+ };
+ if !languages.contains(&language) {
+ languages.push(language);
+ }
+ }
+ languages
+ }
+}
+
+fn prepare_remote(serde_tests: &Path, languages: &[&'static str]) ->
Result<()> {
+ const REVISION: &str = "8193bab45ca5da3b513ea43fb5793e7e5b4da548";
+ let archive_url =
+
format!("https://api.github.com/repos/apache/datasketches-tck/tarball/{REVISION}");
+
+ println!("Downloading serialization snapshots from {archive_url}");
+
+ let timeout = Some(Duration::from_secs(60));
+ let agent: ureq::Agent = ureq::Agent::config_builder()
+ .timeout_connect(timeout)
+ .timeout_recv_response(timeout)
+ .timeout_recv_body(timeout)
+ .build()
+ .into();
+ let response = agent
+ .get(&archive_url)
+ .header("Accept", "application/vnd.github+json")
+ .header("X-GitHub-Api-Version", "2026-03-10")
+ .call()?;
+ let (_, body) = response.into_parts();
+ let mut archive = Archive::new(GzDecoder::new(body.into_reader()));
+
+ let mut targets = vec![];
+ for &language in languages {
+ let directory_name = format!("{language}_generated_files");
+ let source_directory =
Path::new("serialization_test_data").join(&directory_name);
+ let destination = serde_tests.join(directory_name);
+ reset_destination(language, &destination)?;
+ targets.push((language, source_directory, destination, 0_usize));
+ }
+
+ for member in archive.entries()? {
+ let mut member = member?;
+ let path = member.path()?;
+ if !member.header().entry_type().is_file()
+ || path.extension().is_none_or(|extension| extension != "sk")
+ {
+ continue;
+ }
+
+ let Some((_, _, destination, count)) =
targets.iter_mut().find(|target| {
+ path.parent()
+ .is_some_and(|parent| parent.ends_with(&target.1))
+ }) else {
+ continue;
+ };
+
+ let name = path.file_name().expect("snapshot path has a file name");
+ let mut output = fs::File::create(destination.join(name))?;
+ io::copy(&mut member, &mut output)?;
+ *count += 1;
+ }
+
+ for (language, _, destination, count) in targets {
+ if count == 0 {
+ return Err(format!("no {language} snapshots found in the TCK
archive").into());
+ }
+ println!(
+ "Extracted {count} {language} snapshots into {}",
+ destination.display()
+ );
+ }
+ Ok(())
+}
+
+fn prepare_rust(serde_tests: &Path) -> Result<()> {
+ let destination = serde_tests.join("rust_generated_files");
+ reset_destination("rust", &destination)?;
+
+ let mut cmd = find_command("cargo");
+ cmd.args([
+ "run",
+ "--release",
+ "--package",
+ "datagen",
+ "--",
+ "--output-dir",
+ ]);
+ cmd.arg(&destination);
+ println!("{cmd:?}");
+ let status = cmd.status()?;
+ if !status.success() {
+ return Err(format!("datagen failed: {status}").into());
+ }
+
+ let mut count = 0_usize;
+ for entry in fs::read_dir(&destination)? {
+ let entry = entry?;
+ if entry.file_type()?.is_file()
+ && entry
+ .path()
+ .extension()
+ .is_some_and(|extension| extension == "sk")
+ {
+ count += 1;
+ }
+ }
+ if count == 0 {
+ return Err("no rust snapshots were generated".into());
+ }
+ println!(
+ "Generated {count} rust snapshots into {}",
+ destination.display()
+ );
+ Ok(())
+}
+
+fn reset_destination(language: &str, destination: &Path) -> io::Result<()> {
+ if fs::exists(destination)? {
+ println!(
+ "Removing existing {language} snapshots from {}",
+ destination.display()
+ );
+ fs::remove_dir_all(destination)?;
+ }
+ fs::create_dir_all(destination)
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]