This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 5c5a7940755 Hash the full value in fixed-byte BYTES and BIG_DECIMAL
dictionary batch murmur3 reads (#19634)
5c5a7940755 is described below
commit 5c5a79407551579b74efb73b36ff55f2309286d9
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Tue Sep 22 22:10:44 2026 -0700
Hash the full value in fixed-byte BYTES and BIG_DECIMAL dictionary batch
murmur3 reads (#19634)
---
.../local/io/util/FixedByteValueReaderWriter.java | 9 ++
.../pinot/segment/local/io/util/ValueReader.java | 26 ++----
.../local/io/util/VarLengthValueReader.java | 27 +++---
.../index/readers/BaseImmutableDictionary.java | 20 ++---
.../index/readers/BigDecimalDictionary.java | 7 +-
.../segment/index/readers/BytesDictionary.java | 7 +-
.../segment/index/readers/StringDictionary.java | 7 +-
.../index/readers/ImmutableDictionaryTest.java | 97 ++++++++++++++++++++++
8 files changed, 148 insertions(+), 52 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/FixedByteValueReaderWriter.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/FixedByteValueReaderWriter.java
index 7395efb4572..6e62eb268c4 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/FixedByteValueReaderWriter.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/FixedByteValueReaderWriter.java
@@ -92,6 +92,15 @@ public final class FixedByteValueReaderWriter implements
ValueReader {
return new String(buffer, 0, numBytesPerValue, UTF_8);
}
+ @Override
+ public int readBytes(int index, int numBytesPerValue, byte[] buffer) {
+ assert buffer.length >= numBytesPerValue;
+
+ long startOffset = (long) index * numBytesPerValue;
+ _dataBuffer.copyTo(startOffset, buffer, 0, numBytesPerValue);
+ return numBytesPerValue;
+ }
+
@Override
public byte[] getBytes(int index, int numBytesPerValue) {
long startOffset = (long) index * numBytesPerValue;
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/ValueReader.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/ValueReader.java
index 12250cc7778..5e9d34a5940 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/ValueReader.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/ValueReader.java
@@ -22,7 +22,6 @@ import java.io.Closeable;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import org.apache.pinot.spi.utils.BigDecimalUtils;
-import org.apache.pinot.spi.utils.hash.MurmurHashFunctions;
/// Interface for value readers, which read a value at a given index.
@@ -40,7 +39,7 @@ public interface ValueReader extends Closeable {
return BigDecimalUtils.deserialize(getBytes(index, numBytesPerValue));
}
- /// Reads the unpadded bytes into the given buffer and returns the length.
+ /// Reads the unpadded bytes into the given buffer and returns the length.
Applicable to STRING only.
/// NOTE: The passed in reusable buffer should have capacity of at least
`numBytesPerValue`.
int readUnpaddedBytes(int index, int numBytesPerValue, byte[] buffer);
@@ -62,6 +61,11 @@ public interface ValueReader extends Closeable {
/// NOTE: The passed in reusable buffer should have capacity of at least
`numBytesPerValue`.
String getPaddedString(int index, int numBytesPerValue, byte[] buffer);
+ /// Reads the bytes into the given buffer and returns the length. Applicable
to variable sized types other than
+ /// STRING, i.e. BIG_DECIMAL, BYTES.
+ /// NOTE: The passed in reusable buffer should have capacity of at least
`numBytesPerValue`.
+ int readBytes(int index, int numBytesPerValue, byte[] buffer);
+
/// NOTE: Do not reuse buffer for BYTES because the return value can have
variable length.
byte[] getBytes(int index, int numBytesPerValue);
@@ -71,24 +75,6 @@ public interface ValueReader extends Closeable {
/// Applicable to variable sized types other than STRING, i.e. BIG_DECIMAL,
BYTES.
int getByteSize(int index, int numBytesPerValue);
- /// NOTE: The passed in reusable buffer should have capacity of at least
`numBytesPerValue`.
- default int get32BitsMurmur3Hash(int index, int numBytesPerValue, byte[]
buffer) {
- int length = readUnpaddedBytes(index, numBytesPerValue, buffer);
- return MurmurHashFunctions.murmurHash3X64Bit32(buffer, length, 0);
- }
-
- /// NOTE: The passed in reusable buffer should have capacity of at least
`numBytesPerValue`.
- default long get64BitsMurmur3Hash(int index, int numBytesPerValue, byte[]
buffer) {
- int length = readUnpaddedBytes(index, numBytesPerValue, buffer);
- return MurmurHashFunctions.murmurHash3X64Bit64(buffer, length, 0);
- }
-
- /// NOTE: The passed in reusable buffer should have capacity of at least
`numBytesPerValue`.
- default long[] get128BitsMurmur3Hash(int index, int numBytesPerValue, byte[]
buffer) {
- int length = readUnpaddedBytes(index, numBytesPerValue, buffer);
- return MurmurHashFunctions.murmurHash3X64Bit128AsLongs(buffer, length, 0);
- }
-
/// Returns the comparison result of the UTF-8 decoded values.
int compareUtf8Bytes(int index, int numBytesPerValue, byte[] bytes);
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/VarLengthValueReader.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/VarLengthValueReader.java
index 2a485e804f8..c02cef995ec 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/VarLengthValueReader.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/util/VarLengthValueReader.java
@@ -84,17 +84,7 @@ public class VarLengthValueReader implements ValueReader {
@Override
public int readUnpaddedBytes(int index, int numBytesPerValue, byte[] buffer)
{
- assert buffer.length >= numBytesPerValue;
-
- // Read the offset of the byte array first and then read the actual byte
array.
- int offsetPosition = _dataSectionStartOffSet + Integer.BYTES * index;
- int startOffset = _dataBuffer.getInt(offsetPosition);
- int endOffset = _dataBuffer.getInt(offsetPosition + Integer.BYTES);
- int length = endOffset - startOffset;
-
- assert numBytesPerValue >= length;
- _dataBuffer.copyTo(startOffset, buffer, 0, length);
- return length;
+ return readBytes(index, numBytesPerValue, buffer);
}
public void recordOffsetRanges(int index, long baseOffset,
List<ForwardIndexReader.ByteRange> rangeList) {
@@ -111,6 +101,21 @@ public class VarLengthValueReader implements ValueReader {
throw new UnsupportedOperationException();
}
+ @Override
+ public int readBytes(int index, int numBytesPerValue, byte[] buffer) {
+ assert buffer.length >= numBytesPerValue;
+
+ // Read the offset of the byte array first and then read the actual byte
array.
+ int offsetPosition = _dataSectionStartOffSet + Integer.BYTES * index;
+ int startOffset = _dataBuffer.getInt(offsetPosition);
+ int endOffset = _dataBuffer.getInt(offsetPosition + Integer.BYTES);
+ int length = endOffset - startOffset;
+
+ assert numBytesPerValue >= length;
+ _dataBuffer.copyTo(startOffset, buffer, 0, length);
+ return length;
+ }
+
@Override
public byte[] getBytes(int index, int numBytesPerValue) {
// Read the offset of the byte array first and then read the actual byte
array.
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BaseImmutableDictionary.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BaseImmutableDictionary.java
index 8375ef4cbeb..17171b12116 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BaseImmutableDictionary.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BaseImmutableDictionary.java
@@ -251,6 +251,10 @@ public abstract class BaseImmutableDictionary implements
Dictionary {
return _valueReader.getBigDecimal(dictId, _numBytesPerValue);
}
+ protected int readUnpaddedBytes(int dictId, byte[] buffer) {
+ return _valueReader.readUnpaddedBytes(dictId, _numBytesPerValue, buffer);
+ }
+
protected byte[] getUnpaddedBytes(int dictId, byte[] buffer) {
return _valueReader.getUnpaddedBytes(dictId, _numBytesPerValue, buffer);
}
@@ -263,6 +267,10 @@ public abstract class BaseImmutableDictionary implements
Dictionary {
return _valueReader.getPaddedString(dictId, _numBytesPerValue, buffer);
}
+ protected int readBytes(int dictId, byte[] buffer) {
+ return _valueReader.readBytes(dictId, _numBytesPerValue, buffer);
+ }
+
protected byte[] getBytes(int dictId) {
return _valueReader.getBytes(dictId, _numBytesPerValue);
}
@@ -275,18 +283,6 @@ public abstract class BaseImmutableDictionary implements
Dictionary {
return _valueReader.getByteSize(dictId, _numBytesPerValue);
}
- public int get32BitsMurmur3Hash(int dictId, byte[] buffer) {
- return _valueReader.get32BitsMurmur3Hash(dictId, _numBytesPerValue,
buffer);
- }
-
- public long get64BitsMurmur3Hash(int dictId, byte[] buffer) {
- return _valueReader.get64BitsMurmur3Hash(dictId, _numBytesPerValue,
buffer);
- }
-
- public long[] get128BitsMurmur3HashValue(int dictId, byte[] buffer) {
- return _valueReader.get128BitsMurmur3Hash(dictId, _numBytesPerValue,
buffer);
- }
-
protected byte[] getBuffer() {
return new byte[_numBytesPerValue];
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BigDecimalDictionary.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BigDecimalDictionary.java
index e5db75ed499..f4c246cd120 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BigDecimalDictionary.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BigDecimalDictionary.java
@@ -22,6 +22,7 @@ import java.math.BigDecimal;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.BigDecimalUtils;
+import org.apache.pinot.spi.utils.hash.MurmurHashFunctions;
/// Extension of [BaseImmutableDictionary] that implements immutable
dictionary for BigDecimal type.
@@ -105,7 +106,7 @@ public class BigDecimalDictionary extends
BaseImmutableDictionary {
public void read32BitsMurmur3HashValues(int[] dictIds, int length, int[]
outValues) {
byte[] buffer = getBuffer();
for (int i = 0; i < length; i++) {
- outValues[i] = get32BitsMurmur3Hash(dictIds[i], buffer);
+ outValues[i] = MurmurHashFunctions.murmurHash3X64Bit32(buffer,
readBytes(dictIds[i], buffer), 0);
}
}
@@ -113,7 +114,7 @@ public class BigDecimalDictionary extends
BaseImmutableDictionary {
public void read64BitsMurmur3HashValues(int[] dictIds, int length, long[]
outValues) {
byte[] buffer = getBuffer();
for (int i = 0; i < length; i++) {
- outValues[i] = get64BitsMurmur3Hash(dictIds[i], buffer);
+ outValues[i] = MurmurHashFunctions.murmurHash3X64Bit64(buffer,
readBytes(dictIds[i], buffer), 0);
}
}
@@ -121,7 +122,7 @@ public class BigDecimalDictionary extends
BaseImmutableDictionary {
public void read128BitsMurmur3HashValues(int[] dictIds, int length, long[][]
outValues) {
byte[] buffer = getBuffer();
for (int i = 0; i < length; i++) {
- outValues[i] = get128BitsMurmur3HashValue(dictIds[i], buffer);
+ outValues[i] = MurmurHashFunctions.murmurHash3X64Bit128AsLongs(buffer,
readBytes(dictIds[i], buffer), 0);
}
}
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BytesDictionary.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BytesDictionary.java
index 0b1a399454d..0509e5e7aad 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BytesDictionary.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/BytesDictionary.java
@@ -24,6 +24,7 @@ import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.BigDecimalUtils;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.BytesUtils;
+import org.apache.pinot.spi.utils.hash.MurmurHashFunctions;
/// Extension of [BaseImmutableDictionary] that implements immutable
dictionary for byte\[\] type.
@@ -112,7 +113,7 @@ public class BytesDictionary extends
BaseImmutableDictionary {
public void read32BitsMurmur3HashValues(int[] dictIds, int length, int[]
outValues) {
byte[] buffer = getBuffer();
for (int i = 0; i < length; i++) {
- outValues[i] = get32BitsMurmur3Hash(dictIds[i], buffer);
+ outValues[i] = MurmurHashFunctions.murmurHash3X64Bit32(buffer,
readBytes(dictIds[i], buffer), 0);
}
}
@@ -120,7 +121,7 @@ public class BytesDictionary extends
BaseImmutableDictionary {
public void read64BitsMurmur3HashValues(int[] dictIds, int length, long[]
outValues) {
byte[] buffer = getBuffer();
for (int i = 0; i < length; i++) {
- outValues[i] = get64BitsMurmur3Hash(dictIds[i], buffer);
+ outValues[i] = MurmurHashFunctions.murmurHash3X64Bit64(buffer,
readBytes(dictIds[i], buffer), 0);
}
}
@@ -128,7 +129,7 @@ public class BytesDictionary extends
BaseImmutableDictionary {
public void read128BitsMurmur3HashValues(int[] dictIds, int length, long[][]
outValues) {
byte[] buffer = getBuffer();
for (int i = 0; i < length; i++) {
- outValues[i] = get128BitsMurmur3HashValue(dictIds[i], buffer);
+ outValues[i] = MurmurHashFunctions.murmurHash3X64Bit128AsLongs(buffer,
readBytes(dictIds[i], buffer), 0);
}
}
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/StringDictionary.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/StringDictionary.java
index 61e74430176..1c77ef47660 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/StringDictionary.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/StringDictionary.java
@@ -21,6 +21,7 @@ package org.apache.pinot.segment.local.segment.index.readers;
import java.math.BigDecimal;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.utils.hash.MurmurHashFunctions;
public class StringDictionary extends BaseImmutableDictionary {
@@ -179,7 +180,7 @@ public class StringDictionary extends
BaseImmutableDictionary {
public void read32BitsMurmur3HashValues(int[] dictIds, int length, int[]
outValues) {
byte[] buffer = getBuffer();
for (int i = 0; i < length; i++) {
- outValues[i] = get32BitsMurmur3Hash(dictIds[i], buffer);
+ outValues[i] = MurmurHashFunctions.murmurHash3X64Bit32(buffer,
readUnpaddedBytes(dictIds[i], buffer), 0);
}
}
@@ -187,7 +188,7 @@ public class StringDictionary extends
BaseImmutableDictionary {
public void read64BitsMurmur3HashValues(int[] dictIds, int length, long[]
outValues) {
byte[] buffer = getBuffer();
for (int i = 0; i < length; i++) {
- outValues[i] = get64BitsMurmur3Hash(dictIds[i], buffer);
+ outValues[i] = MurmurHashFunctions.murmurHash3X64Bit64(buffer,
readUnpaddedBytes(dictIds[i], buffer), 0);
}
}
@@ -195,7 +196,7 @@ public class StringDictionary extends
BaseImmutableDictionary {
public void read128BitsMurmur3HashValues(int[] dictIds, int length, long[][]
outValues) {
byte[] buffer = getBuffer();
for (int i = 0; i < length; i++) {
- outValues[i] = get128BitsMurmur3HashValue(dictIds[i], buffer);
+ outValues[i] = MurmurHashFunctions.murmurHash3X64Bit128AsLongs(buffer,
readUnpaddedBytes(dictIds[i], buffer), 0);
}
}
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/ImmutableDictionaryTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/ImmutableDictionaryTest.java
index 586c129afe1..b4da5465a45 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/ImmutableDictionaryTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/ImmutableDictionaryTest.java
@@ -27,9 +27,11 @@ import java.io.File;
import java.math.BigDecimal;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
+import java.util.stream.IntStream;
import org.apache.commons.configuration2.PropertiesConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
@@ -373,6 +375,33 @@ public class ImmutableDictionaryTest implements
PinotBuffersAfterMethodCheckRule
assertEquals(bigDecimalDictionary.insertionIndexOf(String.valueOf(randomBigDecimal)),
Arrays.binarySearch(_bigDecimalValues, randomBigDecimal));
}
+ testMurmur3HashValues(bigDecimalDictionary);
+ }
+
+ /// Verifies the murmur3 hash reads of a fixed-byte BIG_DECIMAL dictionary.
Scale 0 values serialize with leading
+ /// `0x00` scale bytes, which are data rather than padding in the fixed-byte
layout.
+ @Test
+ public void testFixedByteBigDecimalDictionaryMurmur3Hash()
+ throws Exception {
+ String columnName = "fixedByteBigDecimalColumn";
+ BigDecimal[] values = new BigDecimal[]{
+ BigDecimal.valueOf(-2), BigDecimal.valueOf(-1), BigDecimal.ZERO,
BigDecimal.ONE, BigDecimal.valueOf(2)
+ };
+ MetricFieldSpec fieldSpec = new MetricFieldSpec(columnName,
DataType.BIG_DECIMAL);
+ fieldSpec.setSingleValueField(true);
+ try (SegmentDictionaryCreator dictionaryCreator = new
SegmentDictionaryCreator(fieldSpec, TEMP_DIR, false)) {
+ dictionaryCreator.build(values);
+ assertEquals(dictionaryCreator.getNumBytesPerEntry(), 3);
+ }
+ try (PinotDataBuffer buffer = PinotDataBuffer.mapReadOnlyBigEndianFile(
+ new File(TEMP_DIR, columnName + V1Constants.Dict.FILE_EXTENSION));
+ BigDecimalDictionary bigDecimalDictionary = new
BigDecimalDictionary(buffer, values.length, 3)) {
+ for (int i = 0; i < values.length; i++) {
+ assertEquals(bigDecimalDictionary.get(i), values[i]);
+ }
+ testMurmur3HashValues(bigDecimalDictionary);
+ testDistinctMurmur3HashValues(bigDecimalDictionary);
+ }
}
@Test
@@ -419,6 +448,7 @@ public class ImmutableDictionaryTest implements
PinotBuffersAfterMethodCheckRule
String randomString = RandomStringUtils.secure().next(RANDOM.nextInt(2 *
MAX_STRING_LENGTH)).replace('\0', ' ');
assertEquals(stringDictionary.insertionIndexOf(randomString),
Arrays.binarySearch(_stringValues, randomString));
}
+ testMurmur3HashValues(stringDictionary);
}
@Test
@@ -468,6 +498,73 @@ public class ImmutableDictionaryTest implements
PinotBuffersAfterMethodCheckRule
assertEquals(bytesDictionary.insertionIndexOf(BytesUtils.toHexString(randomBytes)),
Arrays.binarySearch(_bytesValues, new ByteArray(randomBytes)));
}
+ testMurmur3HashValues(bytesDictionary);
+ }
+
+ /// Verifies the murmur3 hash reads of a fixed-byte BYTES dictionary whose
values contain leading, interior and
+ /// trailing `0x00` bytes, which are data rather than padding in the
fixed-byte layout.
+ @Test
+ public void testFixedByteBytesDictionaryMurmur3Hash()
+ throws Exception {
+ String columnName = "fixedByteBytesColumn";
+ ByteArray[] values = new ByteArray[]{
+ new ByteArray(new byte[]{0, 0, 0, 0}),
+ new ByteArray(new byte[]{0, 0, 0, 1}),
+ new ByteArray(new byte[]{0, 1, 0, 0}),
+ new ByteArray(new byte[]{1, 0, 0, 0}),
+ new ByteArray(new byte[]{1, 0, 0, 1}),
+ new ByteArray(new byte[]{1, 2, 3, 4})
+ };
+ try (SegmentDictionaryCreator dictionaryCreator = new
SegmentDictionaryCreator(
+ new DimensionFieldSpec(columnName, DataType.BYTES, true), TEMP_DIR,
false)) {
+ dictionaryCreator.build(values);
+ assertEquals(dictionaryCreator.getNumBytesPerEntry(), 4);
+ }
+ try (PinotDataBuffer buffer = PinotDataBuffer.mapReadOnlyBigEndianFile(
+ new File(TEMP_DIR, columnName + V1Constants.Dict.FILE_EXTENSION));
+ BytesDictionary bytesDictionary = new BytesDictionary(buffer,
values.length, 4)) {
+ for (int i = 0; i < values.length; i++) {
+ assertEquals(bytesDictionary.get(i), values[i].getBytes());
+ }
+ testMurmur3HashValues(bytesDictionary);
+ testDistinctMurmur3HashValues(bytesDictionary);
+ }
+ }
+
+ /// Asserts that the batch murmur3 hash reads match the single-value reads
for every dict id.
+ private static void testMurmur3HashValues(BaseImmutableDictionary
dictionary) {
+ int length = dictionary.length();
+ int[] dictIds = IntStream.range(0, length).toArray();
+ int[] hashValues32 = new int[length];
+ dictionary.read32BitsMurmur3HashValues(dictIds, length, hashValues32);
+ long[] hashValues64 = new long[length];
+ dictionary.read64BitsMurmur3HashValues(dictIds, length, hashValues64);
+ long[][] hashValues128 = new long[length][];
+ dictionary.read128BitsMurmur3HashValues(dictIds, length, hashValues128);
+ for (int i = 0; i < length; i++) {
+ assertEquals(hashValues32[i], dictionary.get32BitsMurmur3HashValue(i));
+ assertEquals(hashValues64[i], dictionary.get64BitsMurmur3HashValue(i));
+ assertEquals(hashValues128[i], dictionary.get128BitsMurmur3HashValue(i));
+ }
+ }
+
+ /// Asserts that the batch murmur3 hash reads return a distinct hash for
every dict id.
+ private static void testDistinctMurmur3HashValues(BaseImmutableDictionary
dictionary) {
+ int length = dictionary.length();
+ int[] dictIds = IntStream.range(0, length).toArray();
+ int[] hashValues32 = new int[length];
+ dictionary.read32BitsMurmur3HashValues(dictIds, length, hashValues32);
+ long[] hashValues64 = new long[length];
+ dictionary.read64BitsMurmur3HashValues(dictIds, length, hashValues64);
+ long[][] hashValues128 = new long[length][];
+ dictionary.read128BitsMurmur3HashValues(dictIds, length, hashValues128);
+ assertEquals(new IntOpenHashSet(hashValues32).size(), length);
+ assertEquals(new LongOpenHashSet(hashValues64).size(), length);
+ Set<List<Long>> hashSet128 = new HashSet<>();
+ for (long[] hashValue : hashValues128) {
+ hashSet128.add(List.of(hashValue[0], hashValue[1]));
+ }
+ assertEquals(hashSet128.size(), length);
}
/// Regression test for old segments (pre-1.6.0) that have a STRING column
with all-empty values:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
