JingsongLi commented on code in PR #1398:
URL: https://github.com/apache/paimon-rust/pull/1398#discussion_r4656614227


##########
crates/paimon/src/table/goldens/VerifyRustSketchBitmapCompat.java:
##########
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Iterator;
+import java.util.TreeSet;
+
+import org.apache.datasketches.hll.HllSketch;
+import org.apache.datasketches.hll.TgtHllType;
+import org.apache.datasketches.hll.Union;
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.theta.Sketches;
+import org.roaringbitmap.RoaringBitmap;
+import org.roaringbitmap.longlong.Roaring64Bitmap;
+
+/**
+ * JVM-side verifier for Rust-written sketch/bitmap aggregator payloads.
+ *
+ * Uses the same production libraries and call patterns as Paimon utils:
+ * {@code HllSketchUtil}, {@code ThetaSketch}, {@code RoaringBitmap32},
+ * {@code RoaringBitmap64} (DataSketches 4.2.0 / RoaringBitmap 1.2.1).
+ *
+ * Classpath: resolve via {@code pom.xml} in this directory
+ * ({@code mvn dependency:build-classpath}), same versions as paimon-common.
+ *
+ * Args: <goldens-dir> [<rust-written-temp-dir>]
+ */
+public class VerifyRustSketchBitmapCompat {
+
+    private static int failures = 0;
+
+    public static void main(String[] args) throws Exception {
+        Path goldens = Path.of(args[0]);
+        Path rustOut = Path.of(args.length > 1 ? args[1] : 
goldens.resolve("rust_written").toString());
+
+        verifyRbm32(goldens, rustOut);
+        verifyRbm64(goldens, rustOut);
+        verifyHll(goldens, rustOut);
+        verifyTheta(goldens, rustOut);
+
+        if (failures != 0) {
+            System.err.println("FAILED checks=" + failures);
+            System.exit(1);
+        }
+        System.out.println("OK all JVM compatibility checks passed");
+    }
+
+    /** Mirrors FieldRoaringBitmap32Agg / RoaringBitmap32.deserialize + or. */
+    private static void verifyRbm32(Path goldens, Path rustOut) throws 
Exception {
+        RoaringBitmap javaA = readRbm32(goldens.resolve("compat_rbm32_a.bin"));
+        RoaringBitmap javaB = readRbm32(goldens.resolve("compat_rbm32_b.bin"));
+        RoaringBitmap javaUnion = 
readRbm32(goldens.resolve("compat_rbm32_union.bin"));
+
+        RoaringBitmap merged = javaA.clone();
+        merged.or(javaB);
+        check("rbm32 java∪java set", 
values32(merged).equals(values32(javaUnion)));
+
+        // Rust-written operands must deserialize with Java RoaringBitmap.
+        RoaringBitmap rustA = readRbm32(rustOut.resolve("rust_rbm32_a.bin"));
+        RoaringBitmap rustB = readRbm32(rustOut.resolve("rust_rbm32_b.bin"));
+        RoaringBitmap rustUnion = 
readRbm32(rustOut.resolve("rust_rbm32_union.bin"));
+        check("rbm32 rust_a set", values32(rustA).equals(values32(javaA)));
+        check("rbm32 rust_b set", values32(rustB).equals(values32(javaB)));
+        check("rbm32 rust_union set", 
values32(rustUnion).equals(values32(javaUnion)));
+
+        RoaringBitmap cross = rustA.clone();
+        cross.or(javaB);
+        check("rbm32 rust∪java set", 
values32(cross).equals(values32(javaUnion)));
+
+        RoaringBitmap cross2 = javaA.clone();
+        cross2.or(rustB);
+        check("rbm32 java∪rust set", 
values32(cross2).equals(values32(javaUnion)));
+
+        RoaringBitmap empty = 
readRbm32(goldens.resolve("compat_rbm32_empty.bin"));
+        RoaringBitmap dups = 
readRbm32(goldens.resolve("compat_rbm32_dups.bin"));
+        RoaringBitmap bounds = 
readRbm32(goldens.resolve("compat_rbm32_bounds.bin"));
+        check("rbm32 empty", empty.getCardinality() == 0);
+        check("rbm32 dups", values32(dups).equals(set32(7, 8)));
+        check(
+                "rbm32 bounds",
+                values32(bounds)
+                        .equals(set32(0, Integer.MAX_VALUE, 0xFFFFFFFF)));
+    }
+
+    /** Mirrors FieldRoaringBitmap64Agg / RoaringBitmap64.deserialize + or. */
+    private static void verifyRbm64(Path goldens, Path rustOut) throws 
Exception {
+        Roaring64Bitmap javaA = 
readRbm64(goldens.resolve("compat_rbm64_a.bin"));
+        Roaring64Bitmap javaB = 
readRbm64(goldens.resolve("compat_rbm64_b.bin"));
+        Roaring64Bitmap javaUnion = 
readRbm64(goldens.resolve("compat_rbm64_union.bin"));
+
+        Roaring64Bitmap merged = javaA.clone();
+        merged.or(javaB);
+        check("rbm64 java∪java card", merged.getLongCardinality() == 
javaUnion.getLongCardinality());
+        check("rbm64 java∪java set", 
values64(merged).equals(values64(javaUnion)));
+
+        Roaring64Bitmap rustA = readRbm64(rustOut.resolve("rust_rbm64_a.bin"));
+        Roaring64Bitmap rustB = readRbm64(rustOut.resolve("rust_rbm64_b.bin"));
+        Roaring64Bitmap rustUnion = 
readRbm64(rustOut.resolve("rust_rbm64_union.bin"));
+        check("rbm64 rust_a set", values64(rustA).equals(values64(javaA)));
+        check("rbm64 rust_b set", values64(rustB).equals(values64(javaB)));
+        check("rbm64 rust_union set", 
values64(rustUnion).equals(values64(javaUnion)));
+
+        Roaring64Bitmap cross = rustA.clone();
+        cross.or(javaB);
+        check("rbm64 rust∪java set", 
values64(cross).equals(values64(javaUnion)));
+
+        Roaring64Bitmap cross2 = javaA.clone();
+        cross2.or(rustB);
+        check("rbm64 java∪rust set", 
values64(cross2).equals(values64(javaUnion)));
+
+        Roaring64Bitmap empty = 
readRbm64(goldens.resolve("compat_rbm64_empty.bin"));
+        check("rbm64 empty", empty.getLongCardinality() == 0);
+    }
+
+    /** Mirrors HllSketchUtil.union (heapify + Union + compact HLL_4). */
+    private static void verifyHll(Path goldens, Path rustOut) throws Exception 
{
+        byte[] javaA = Files.readAllBytes(goldens.resolve("compat_hll_a.bin"));
+        byte[] javaB = Files.readAllBytes(goldens.resolve("compat_hll_b.bin"));
+        byte[] javaUnion = 
Files.readAllBytes(goldens.resolve("compat_hll_union.bin"));
+        double expected = HllSketch.heapify(javaUnion).getEstimate();
+
+        double javaMerged = hllUnionEstimate(javaA, javaB);
+        check("hll java∪java estimate", Math.abs(javaMerged - expected) < 
1e-9);
+
+        byte[] rustUnion = 
Files.readAllBytes(rustOut.resolve("rust_hll_union.bin"));
+        // Production Java deserialize of Rust-written compact HLL.
+        double rustAsJava = HllSketch.heapify(rustUnion).getEstimate();
+        check("hll rust-bytes via Java heapify", Math.abs(rustAsJava - 
expected) < 1e-6);
+
+        double rustThenJava = hllUnionEstimate(rustUnion, javaA);

Review Comment:
   **[P2] Check the B-only contribution in the JVM HLL merge assertion**
   
   The Rust payload is meant to contain `A ∪ B`, but the only mixed-language 
merge re-adds `A`. I swapped in a valid fixture containing `A` plus 100 
unrelated values in place of the 100 values unique to `B`. The complete Java 
verifier still passed: both the standalone and re-add-A estimates were 
`300.000222772550`. With the original `B`, that same fixture estimates 
`399.919387542587`, while the actual Rust output stays at `300.000222772550`. 
Add a Java merge with `B` (or with a probe overlapping its unique elements) and 
compare against the Java reference. This catches a semantic regression that 
would otherwise inflate downstream mixed-writer distinct counts even though the 
payload deserializes and its initial estimate matches.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to