This is an automated email from the ASF dual-hosted git repository. leerho pushed a commit to branch check_cross_lang_hll_binary in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit a9954b988bd610c674458083dfcaba2cad172db3 Author: Lee Rhodes <[email protected]> AuthorDate: Thu Sep 24 16:20:27 2026 -0700 Changes to allow cross-language binaries to be the same as C++. --- .../datasketches/theta/EmptyCompactSketch.java | 21 ++++++++--------- .../apache/datasketches/theta/PreambleUtil.java | 12 +++++----- .../org/apache/datasketches/common/UtilityIO.java | 2 +- .../hll/HllSketchCrossLanguageTest.java | 26 ++++++++++++++++++++++ .../theta/ThetaSketchCrossLanguageTest.java | 21 +++++++++++++++++ 5 files changed, 65 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java b/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java index a5ffed2f7..1ebc47afe 100644 --- a/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java +++ b/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java @@ -27,29 +27,30 @@ import java.lang.foreign.MemorySegment; import org.apache.datasketches.common.SketchesArgumentException; /** - * Singleton empty CompactThetaSketch. + * Singleton Empty CompactThetaSketch. * * @author Lee Rhodes */ final class EmptyCompactSketch extends CompactThetaSketch { - //For backward compatibility, a candidate long must have Flags= compact, read-only, - // COMPACT-Family=3, SerVer=3, PreLongs=1, and be exactly 8 bytes long. The seedHash is ignored. - // NOTE: The empty and ordered flags may or may not be set - private static final long EMPTY_SKETCH_MASK = 0X00_00_EB_00_00_FF_FF_FFL; - private static final long EMPTY_SKETCH_TEST = 0X00_00_0A_00_00_03_03_01L; - //When returning a byte array the empty and ordered bits are also set + // For backward compatibility, a candidate EmptyCompactSketch + // must have Flags: compact, empty, read-only; + // and bytes: COMPACT-Family=3, SerVer=3, PreLongs=1; and be exactly 8 bytes long. + // NOTE: The seedHash bytes are ignored + private static final long EMPTY_SKETCH_MASK = 0X00_00_FF_FF_FF_FF_FF_FFL; + private static final long EMPTY_SKETCH_TEST = 0X00_00_1E_00_00_03_03_01L; + //When returning a byte array the following flags are set: ordered, compact, empty, read-only static final byte[] EMPTY_COMPACT_SKETCH_ARR = { 1, 3, 3, 0, 0, 0x1E, 0, 0 }; private static final EmptyCompactSketch EMPTY_COMPACT_SKETCH = new EmptyCompactSketch(); private EmptyCompactSketch() {} - static synchronized EmptyCompactSketch getInstance() { + static EmptyCompactSketch getInstance() { return EMPTY_COMPACT_SKETCH; } - //This should be a heapify - static synchronized EmptyCompactSketch getHeapInstance(final MemorySegment srcSeg) { + //This is effectively a heapify + static EmptyCompactSketch getHeapInstance(final MemorySegment srcSeg) { final long pre0 = srcSeg.get(JAVA_LONG_UNALIGNED, 0); if (testCandidatePre0(pre0)) { return EMPTY_COMPACT_SKETCH; diff --git a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java index fcb7f24a4..f8239511f 100644 --- a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java +++ b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java @@ -204,12 +204,12 @@ final class PreambleUtil { static final int UNION_THETA_LONG = 24; //8-byte aligned, only used by ThetaUnion // flag byte bit masks - static final int RESERVED_FLAG_MASK = 1; //Bit 0: Reserved, no longer used. Was BigEndian - static final int READ_ONLY_FLAG_MASK = 2; //Bit 1: Reserved, Set but not read. - static final int EMPTY_FLAG_MASK = 4; //Bit 2: - static final int COMPACT_FLAG_MASK = 8; //Bit 3: - static final int ORDERED_FLAG_MASK = 16;//Bit 4: - static final int SINGLEITEM_FLAG_MASK = 32;//Bit 5: + static final int RESERVED_FLAG_MASK = 1; //Bit 0: 0x1: Reserved, no longer used. Was BigEndian + static final int READ_ONLY_FLAG_MASK = 2; //Bit 1: 0x2: + static final int EMPTY_FLAG_MASK = 4; //Bit 2: 0x4: + static final int COMPACT_FLAG_MASK = 8; //Bit 3: 0x8: + static final int ORDERED_FLAG_MASK = 16;//Bit 4: 0x10: + static final int SINGLEITEM_FLAG_MASK = 32;//Bit 5: 0x20: //The last 2 bits (Bit 6,7) of the flags byte are reserved and assumed to be zero. //Other constants diff --git a/src/test/java/org/apache/datasketches/common/UtilityIO.java b/src/test/java/org/apache/datasketches/common/UtilityIO.java index 923dc20c2..eb8ff6a94 100644 --- a/src/test/java/org/apache/datasketches/common/UtilityIO.java +++ b/src/test/java/org/apache/datasketches/common/UtilityIO.java @@ -31,7 +31,7 @@ import java.util.Objects; * Utilities common to testing */ public final class UtilityIO { - private static final String LS = System.getProperty("line.separator"); + public static final String LS = System.getProperty("line.separator"); private static final Class<?> clazz = UtilityIO.class; private static final ClassLoader CL = clazz.getClassLoader(); private static final String TEST_DATA_ROOT_PROP = "test.data.root"; diff --git a/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java index 8595391a4..84b3af145 100644 --- a/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java @@ -23,6 +23,7 @@ import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.LS; import static org.apache.datasketches.common.UtilityIO.getFileBytes; import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.apache.datasketches.hll.TgtHllType.HLL_4; @@ -34,6 +35,7 @@ import static org.testng.Assert.assertTrue; import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; @@ -68,16 +70,20 @@ public class HllSketchCrossLanguageTest { @Test(groups = {CHECK_JAVA_FILES}, priority = 1) public void checkJava() { deserializeHll(GroupLanguage.JAVA); + deserializeHll_crossBinary(GroupLanguage.JAVA, GroupLanguage.CPP); + } @Test(groups = {CHECK_CPP_FILES}) public void checkCpp() { deserializeHll(GroupLanguage.CPP); + deserializeHll_crossBinary(GroupLanguage.JAVA, GroupLanguage.CPP); } @Test(groups = {CHECK_GO_FILES}) public void checkGo() { deserializeHll(GroupLanguage.GO); + deserializeHll_crossBinary(GroupLanguage.JAVA, GroupLanguage.GO); } private static void deserializeHll(final GroupLanguage lang) { @@ -97,4 +103,24 @@ public class HllSketchCrossLanguageTest { } } + private static void deserializeHll_crossBinary(final GroupLanguage lang1, final GroupLanguage lang2) { + final String[] sArr = {"hll4", "hll6", "hll8"}; + final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; + for (final String s: sArr) { + for (final int n: nArr) { + final String fileName1 = s + "_n" + n + lang1.sfx + ".sk"; + final byte[] bytes1 = getFileBytes(lang1.pth, fileName1); + final String fileName2 = s + "_n" + n + lang2.sfx + ".sk"; + final byte[] bytes2 = getFileBytes(lang2.pth, fileName2); + if (bytes1.length == 0 || bytes2.length == 0) { continue;} + try { assertEquals(bytes1, bytes2); } + catch (final AssertionError e) { + String err = "FileName1: " + fileName1 + ", FileName2: " + fileName2 + + ", HllType: " + s + ", n: " + n; + throw new SketchesArgumentException(err + LS + e); + } + } + } + } + } diff --git a/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java index 5aba332ff..9d64f201f 100644 --- a/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java @@ -23,6 +23,7 @@ import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.LS; import static org.apache.datasketches.common.UtilityIO.getFileBytes; import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; @@ -32,6 +33,7 @@ import static org.testng.Assert.assertTrue; import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; @@ -92,6 +94,7 @@ public class ThetaSketchCrossLanguageTest { deserializeCompressedFromFile(GroupLanguage.CPP); deserializeNonEmptyNoEntriesUsingSegment(GroupLanguage.CPP); deserializeNonEmptyNoEntriesFromFile(GroupLanguage.CPP); + deserializeSketchesFromFile_crossBinary(GroupLanguage.CPP, GroupLanguage.JAVA); } @Test(groups = {CHECK_GO_FILES}) @@ -102,6 +105,7 @@ public class ThetaSketchCrossLanguageTest { deserializeCompressedFromFile(GroupLanguage.GO); deserializeNonEmptyNoEntriesUsingSegment(GroupLanguage.GO); deserializeNonEmptyNoEntriesFromFile(GroupLanguage.GO); + //deserializeSketchesFromFile_crossBinary(GroupLanguage.GO, GroupLanguage.JAVA); } private static void deserializeSketchesUsingSegment(final GroupLanguage lang) { @@ -146,6 +150,23 @@ public class ThetaSketchCrossLanguageTest { } } + private static void deserializeSketchesFromFile_crossBinary(final GroupLanguage lang1, final GroupLanguage lang2) { + final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; + for (final int n: nArr) { + final String fileName1 = "theta_n" + n + lang1.sfx + ".sk"; + final byte[] bytes1 = getFileBytes(lang1.pth, fileName1); + final String fileName2 = "theta_n" + n + lang2.sfx + ".sk"; + final byte[] bytes2 = getFileBytes(lang2.pth, fileName2); + if (bytes1.length == 0 || bytes2.length == 0) { continue; } + try { assertEquals(bytes1, bytes2); } + catch (final AssertionError e) { + String err = "FileName1: " + fileName1 + ", FileName2: " + fileName2 + + ", n: " + n; + throw new SketchesArgumentException(err + LS + e); + } + } + } + private static void deserializeCompressedUsingSegment(final GroupLanguage lang) { final int[] nArr = {10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
