benwtrent commented on code in PR #15564:
URL: https://github.com/apache/lucene/pull/15564#discussion_r2686153260
##########
lucene/core/src/java/org/apache/lucene/util/quantization/OptimizedScalarQuantizer.java:
##########
@@ -440,6 +440,86 @@ public static void unpackBinary(byte[] packed, byte[]
vector) {
}
}
+ /**
+ * Transpose a 2-bit (dibit) quantized vector into a byte array for
efficient bitwise operations.
+ * The result has 2 stripes: similar to {@link #transposeHalfByte}, but only
for 2 bits
+ *
+ * @param vector the 2-bit quantized vector (values 0-3)
+ * @param packed the byte array to store the transposed vector
+ */
+ public static void transposeDibit(byte[] vector, byte[] packed) {
+ int limit = vector.length - 7;
+ int i = 0;
+ int index = 0;
+ for (; i < limit; i += 8, index++) {
+ int lowerByte =
+ (vector[i] & 1) << 7
+ | (vector[i + 1] & 1) << 6
+ | (vector[i + 2] & 1) << 5
+ | (vector[i + 3] & 1) << 4
+ | (vector[i + 4] & 1) << 3
+ | (vector[i + 5] & 1) << 2
+ | (vector[i + 6] & 1) << 1
+ | (vector[i + 7] & 1);
+ int upperByte =
+ ((vector[i] >> 1) & 1) << 7
+ | ((vector[i + 1] >> 1) & 1) << 6
+ | ((vector[i + 2] >> 1) & 1) << 5
+ | ((vector[i + 3] >> 1) & 1) << 4
+ | ((vector[i + 4] >> 1) & 1) << 3
+ | ((vector[i + 5] >> 1) & 1) << 2
+ | ((vector[i + 6] >> 1) & 1) << 1
+ | ((vector[i + 7] >> 1) & 1);
+ packed[index] = (byte) lowerByte;
+ packed[index + packed.length / 2] = (byte) upperByte;
+ }
+ if (i == vector.length) {
+ return;
+ }
+ int lowerByte = 0;
+ int upperByte = 0;
+ for (int j = 7; i < vector.length; j--, i++) {
+ assert vector[i] >= 0 && vector[i] <= 3;
+ lowerByte |= (vector[i] & 1) << j;
+ upperByte |= ((vector[i] >> 1) & 1) << j;
+ }
+ packed[index] = (byte) lowerByte;
+ packed[index + packed.length / 2] = (byte) upperByte;
+ }
+
+ /**
Review Comment:
Untranspose is used in the event of floating points not being available and
thus we need to get the original values back and dequantize.
See
lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedFloatVectorValues.java
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
