paul-rogers commented on a change in pull request #11853:
URL: https://github.com/apache/druid/pull/11853#discussion_r738895046
##########
File path: core/src/main/java/org/apache/druid/segment/column/Types.java
##########
@@ -112,4 +121,529 @@
return (typeSignature1 != null && typeSignature1.is(typeDescriptor)) ||
(typeSignature2 != null && typeSignature2.is(typeDescriptor));
}
+
+ /**
+ * Get an {@link ObjectByteStrategy} registered to some {@link
TypeSignature#getComplexTypeName()}.
+ */
+ @Nullable
+ public static ObjectByteStrategy<?> getStrategy(String type)
+ {
+ return STRATEGIES.get(type);
+ }
+
+ /**
+ * hmm... this might look familiar... (see ComplexMetrics)
+ *
+ * Register a complex type name -> {@link ObjectByteStrategy} mapping.
+ *
+ * If the specified type name or type id are already used and the supplied
{@link ObjectByteStrategy} is not of the
+ * same type as the existing value in the map for said key, an {@link ISE}
is thrown.
+ *
+ * @param strategy The {@link ObjectByteStrategy} object to be associated
with the 'type' in the map.
+ */
+ public static void registerStrategy(String typeName, ObjectByteStrategy<?>
strategy)
+ {
+ Preconditions.checkNotNull(typeName);
+ STRATEGIES.compute(typeName, (key, value) -> {
+ if (value == null) {
+ return strategy;
+ } else {
+ if (!value.getClass().getName().equals(strategy.getClass().getName()))
{
+ throw new ISE(
+ "Incompatible strategy for type[%s] already exists. Expected
[%s], found [%s].",
+ key,
+ strategy.getClass().getName(),
+ value.getClass().getName()
+ );
+ } else {
+ return value;
+ }
+ }
+ });
+ }
+
+ /**
+ * Clear and set the 'null' byte of a nullable value to {@link
NullHandling#IS_NULL_BYTE} to a {@link ByteBuffer} at
+ * the supplied position. This method does not change the buffer position,
limit, or mark.
+ *
+ * Nullable types are stored with a leading byte to indicate if the value is
null, followed by teh value bytes
Review comment:
teh -> the
##########
File path: core/src/main/java/org/apache/druid/math/expr/ExprEval.java
##########
@@ -26,138 +26,56 @@
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.UOE;
+import org.apache.druid.segment.column.ObjectByteStrategy;
+import org.apache.druid.segment.column.Types;
import javax.annotation.Nullable;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
-import java.util.Objects;
/**
* Generic result holder for evaluated {@link Expr} containing the value and
{@link ExprType} of the value to allow
*/
public abstract class ExprEval<T>
{
- private static final int NULL_LENGTH = -1;
-
- /**
- * Deserialize an expression stored in a bytebuffer, e.g. for an agg.
- *
- * This should be refactored to be consolidated with some of the standard
type handling of aggregators probably
- */
- public static ExprEval deserialize(ByteBuffer buffer, int position)
- {
- final ExprType type = ExprType.fromByte(buffer.get(position));
- return deserialize(buffer, position + 1, type);
- }
-
/**
* Deserialize an expression stored in a bytebuffer, e.g. for an agg.
*
* This should be refactored to be consolidated with some of the standard
type handling of aggregators probably
*/
- public static ExprEval deserialize(ByteBuffer buffer, int offset, ExprType
type)
+ public static ExprEval deserialize(ByteBuffer buffer, int offset,
ExpressionType type)
{
- // | expression bytes |
- switch (type) {
+ switch (type.getType()) {
case LONG:
- // | expression type (byte) | is null (byte) | long bytes |
- if (buffer.get(offset++) == NullHandling.IS_NOT_NULL_BYTE) {
- return of(buffer.getLong(offset));
+ if (Types.isNullableNull(buffer, offset)) {
+ return ofLong(null);
}
- return ofLong(null);
+ return of(Types.readNullableLong(buffer, offset));
case DOUBLE:
- // | expression type (byte) | is null (byte) | double bytes |
- if (buffer.get(offset++) == NullHandling.IS_NOT_NULL_BYTE) {
- return of(buffer.getDouble(offset));
+ if (Types.isNullableNull(buffer, offset)) {
+ return ofDouble(null);
}
- return ofDouble(null);
+ return of(Types.readNullableDouble(buffer, offset));
case STRING:
- // | expression type (byte) | string length (int) | string bytes |
- final int length = buffer.getInt(offset);
- if (length < 0) {
+ if (Types.isNullableNull(buffer, offset)) {
return of(null);
}
- final byte[] stringBytes = new byte[length];
- final int oldPosition = buffer.position();
- buffer.position(offset + Integer.BYTES);
- buffer.get(stringBytes, 0, length);
- buffer.position(oldPosition);
+ final byte[] stringBytes = Types.readNullableVariableBlob(buffer,
offset);
return of(StringUtils.fromUtf8(stringBytes));
case ARRAY:
- final ExprType elementType = ExprType.fromByte(buffer.get(offset++));
- switch (elementType) {
+ switch (type.getElementType().getType()) {
case LONG:
- // | expression type (byte) | array element type (byte) | array
length (int) | array bytes |
- final int longArrayLength = buffer.getInt(offset);
- offset += Integer.BYTES;
- if (longArrayLength < 0) {
- return ofLongArray(null);
- }
- final Long[] longs = new Long[longArrayLength];
- for (int i = 0; i < longArrayLength; i++) {
- final byte isNull = buffer.get(offset);
- offset += Byte.BYTES;
- if (isNull == NullHandling.IS_NOT_NULL_BYTE) {
- // | is null (byte) | long bytes |
- longs[i] = buffer.getLong(offset);
- offset += Long.BYTES;
- } else {
- // | is null (byte) |
- longs[i] = null;
- }
- }
- return ofLongArray(longs);
+ return ofLongArray(Types.readNullableLongArray(buffer, offset));
case DOUBLE:
- // | expression type (byte) | array element type (byte) | array
length (int) | array bytes |
- final int doubleArrayLength = buffer.getInt(offset);
- offset += Integer.BYTES;
- if (doubleArrayLength < 0) {
- return ofDoubleArray(null);
- }
- final Double[] doubles = new Double[doubleArrayLength];
- for (int i = 0; i < doubleArrayLength; i++) {
- final byte isNull = buffer.get(offset);
- offset += Byte.BYTES;
- if (isNull == NullHandling.IS_NOT_NULL_BYTE) {
- // | is null (byte) | double bytes |
- doubles[i] = buffer.getDouble(offset);
- offset += Double.BYTES;
- } else {
- // | is null (byte) |
- doubles[i] = null;
- }
- }
- return ofDoubleArray(doubles);
+ return ofDoubleArray(Types.readNullableDoubleArray(buffer,
offset));
case STRING:
- // | expression type (byte) | array element type (byte) | array
length (int) | array bytes |
- final int stringArrayLength = buffer.getInt(offset);
- offset += Integer.BYTES;
- if (stringArrayLength < 0) {
- return ofStringArray(null);
- }
- final String[] stringArray = new String[stringArrayLength];
- for (int i = 0; i < stringArrayLength; i++) {
- final int stringElementLength = buffer.getInt(offset);
- offset += Integer.BYTES;
- if (stringElementLength < 0) {
- // | string length (int) |
- stringArray[i] = null;
- } else {
- // | string length (int) | string bytes |
- final byte[] stringElementBytes = new
byte[stringElementLength];
- final int oldPosition2 = buffer.position();
- buffer.position(offset);
- buffer.get(stringElementBytes, 0, stringElementLength);
- buffer.position(oldPosition2);
- stringArray[i] = StringUtils.fromUtf8(stringElementBytes);
- offset += stringElementLength;
- }
- }
- return ofStringArray(stringArray);
+ return ofStringArray(Types.readNullableStringArray(buffer,
offset));
Review comment:
Nice simplification.
##########
File path: core/src/main/java/org/apache/druid/segment/column/Types.java
##########
@@ -112,4 +121,529 @@
return (typeSignature1 != null && typeSignature1.is(typeDescriptor)) ||
(typeSignature2 != null && typeSignature2.is(typeDescriptor));
}
+
+ /**
+ * Get an {@link ObjectByteStrategy} registered to some {@link
TypeSignature#getComplexTypeName()}.
+ */
+ @Nullable
+ public static ObjectByteStrategy<?> getStrategy(String type)
+ {
+ return STRATEGIES.get(type);
+ }
+
+ /**
+ * hmm... this might look familiar... (see ComplexMetrics)
+ *
+ * Register a complex type name -> {@link ObjectByteStrategy} mapping.
+ *
+ * If the specified type name or type id are already used and the supplied
{@link ObjectByteStrategy} is not of the
+ * same type as the existing value in the map for said key, an {@link ISE}
is thrown.
+ *
+ * @param strategy The {@link ObjectByteStrategy} object to be associated
with the 'type' in the map.
+ */
Review comment:
If we like our Javadoc to be Javadoc-like, might want to add `<p>` to
mark paragraphs. Else, it all gets run together.
##########
File path: core/src/main/java/org/apache/druid/segment/column/Types.java
##########
@@ -112,4 +121,529 @@
return (typeSignature1 != null && typeSignature1.is(typeDescriptor)) ||
(typeSignature2 != null && typeSignature2.is(typeDescriptor));
}
+
+ /**
+ * Get an {@link ObjectByteStrategy} registered to some {@link
TypeSignature#getComplexTypeName()}.
+ */
+ @Nullable
+ public static ObjectByteStrategy<?> getStrategy(String type)
+ {
+ return STRATEGIES.get(type);
+ }
+
+ /**
+ * hmm... this might look familiar... (see ComplexMetrics)
+ *
+ * Register a complex type name -> {@link ObjectByteStrategy} mapping.
+ *
+ * If the specified type name or type id are already used and the supplied
{@link ObjectByteStrategy} is not of the
+ * same type as the existing value in the map for said key, an {@link ISE}
is thrown.
+ *
+ * @param strategy The {@link ObjectByteStrategy} object to be associated
with the 'type' in the map.
+ */
+ public static void registerStrategy(String typeName, ObjectByteStrategy<?>
strategy)
+ {
+ Preconditions.checkNotNull(typeName);
+ STRATEGIES.compute(typeName, (key, value) -> {
+ if (value == null) {
+ return strategy;
+ } else {
+ if (!value.getClass().getName().equals(strategy.getClass().getName()))
{
+ throw new ISE(
+ "Incompatible strategy for type[%s] already exists. Expected
[%s], found [%s].",
+ key,
+ strategy.getClass().getName(),
+ value.getClass().getName()
+ );
+ } else {
+ return value;
+ }
+ }
+ });
+ }
+
+ /**
+ * Clear and set the 'null' byte of a nullable value to {@link
NullHandling#IS_NULL_BYTE} to a {@link ByteBuffer} at
+ * the supplied position. This method does not change the buffer position,
limit, or mark.
+ *
+ * Nullable types are stored with a leading byte to indicate if the value is
null, followed by teh value bytes
+ * (if not null)
+ *
+ * layout: | null (byte) | value |
+ *
+ * @return number of bytes written (always 1)
+ */
+ public static int writeNull(ByteBuffer buffer, int offset)
+ {
+ buffer.put(offset, NullHandling.IS_NULL_BYTE);
+ return 1;
+ }
+
+ /**
+ * Checks if a 'nullable' value's null byte is set to {@link
NullHandling#IS_NULL_BYTE}. This method will mask the
+ * value of the null byte to only check if the null bit is set, meaning that
the higher bits can be utilized for
+ * flags as necessary (e.g. using high bits to indicate if the value has
been set or not for aggregators).
+ *
+ * Note that writing nullable values with the methods of {@link Types} will
always clear and set the null byte to
+ * either {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE}, losing any flag bits.
+ *
+ * layout: | null (byte) | value |
+ */
+ public static boolean isNullableNull(ByteBuffer buffer, int offset)
+ {
+ // use & so that callers can use the high bits of the null byte to pack
additional information if necessary
+ return (buffer.get(offset) & NullHandling.IS_NULL_BYTE) ==
NullHandling.IS_NULL_BYTE;
+ }
+
+ /**
+ * Write a non-null long value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the long value is written
in the next 8 bytes.
+ *
+ * layout: | null (byte) | long |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 9)
+ */
+ public static int writeNullableLong(ByteBuffer buffer, int offset, long
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putLong(offset, value);
+ return NULLABLE_LONG_SIZE;
+ }
+
+ /**
+ * Reads a non-null long value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | long |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static long readNullableLong(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getLong(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a non-null double value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the double value is written
in the next 8 bytes.
+ *
+ * layout: | null (byte) | double |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 9)
+ */
+ public static int writeNullableDouble(ByteBuffer buffer, int offset, double
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putDouble(offset, value);
+ return NULLABLE_DOUBLE_SIZE;
+ }
+
+ /**
+ * Reads a non-null double value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | double |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static double readNullableDouble(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getDouble(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a non-null float value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the float value is written
in the next 4 bytes.
+ *
+ * layout: | null (byte) | float |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 5)
+ */
+ public static int writeNullableFloat(ByteBuffer buffer, int offset, float
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putFloat(offset, value);
+ return NULLABLE_FLOAT_SIZE;
+ }
+
+ /**
+ * Reads a non-null float value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | float |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static float readNullableFloat(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getFloat(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a variably lengthed byte[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the byte[] value
+ * is not null, the size in bytes is written as an integer in the next 4
bytes, followed by the byte[] value itself.
+ *
+ * layout: | null (byte) | size (int) | byte[] |
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of byte[] if not)
+ */
+ public static int writeNullableVariableBlob(ByteBuffer buffer, int offset,
@Nullable byte[] value)
+ {
+ // | null (byte) | length (int) | bytes |
+ final int size;
+ if (value != null) {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putInt(offset, value.length);
+ offset += Integer.BYTES;
+ final int oldPosition = buffer.position();
+ buffer.position(offset);
+ buffer.put(value, 0, value.length);
+ buffer.position(oldPosition);
+ size = 1 + Integer.BYTES + value.length;
+ } else {
+ size = writeNull(buffer, offset);
+ }
+ return size;
+ }
+
+ /**
+ * Reads a nullable variably lengthed byte[] value from a {@link ByteBuffer}
at the supplied offset. If the null byte
+ * is set to {@link NullHandling#IS_NULL_BYTE}, this method will return
null, else it will read the next 4 bytes to
+ * get the byte[] size followed by that many bytes to extract the value.
+ *
+ * layout: | null (byte) | size (int) | byte[] |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ @Nullable
+ public static byte[] readNullableVariableBlob(ByteBuffer buffer, int offset)
+ {
+ // | null (byte) | length (int) | bytes |
+ final int length = buffer.getInt(offset + VALUE_OFFSET);
+ final byte[] blob = new byte[length];
+ final int oldPosition = buffer.position();
+ buffer.position(offset + VALUE_OFFSET + Integer.BYTES);
+ buffer.get(blob, 0, length);
+ buffer.position(oldPosition);
+ return blob;
+ }
+
+ /**
+ * Write a variably lengthed Long[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the Long[] value
+ * is not null, the size in bytes is written as an integer in the next 4
bytes. Elements of the array are each written
+ * out with {@link #writeNull} if null, or {@link #writeNullableLong} if
not, taking either 1 or 9 bytes each. If the
+ * total byte size of serializing the array is larger than the max size
parameter, this method will explode via a call
+ * to {@link #checkMaxBytes}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | long |, | null
(byte) |, ... |null (byte) | long |} |
+ *
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of Long[] if not)
+ */
+ public static int writeNullableLongArray(ByteBuffer buffer, int offset,
@Nullable Long[] array, int maxSizeBytes)
Review comment:
How do we ensure that the buffer has capacity? In the prior types,
lengths are known, so code could check if there is capacity. For the array (and
complex type), how do we prevent buffer exhaustion given we don't (easily) know
the required capacity ahead of time?
##########
File path: core/src/test/java/org/apache/druid/math/expr/ExprEvalTest.java
##########
@@ -143,10 +151,10 @@ public void testLongArrayEvalTooBig()
expectedException.expectMessage(StringUtils.format(
"Unable to serialize [%s], size [%s] is larger than max [%s]",
ExpressionType.LONG_ARRAY,
- NullHandling.sqlCompatible() ? 33 : 30,
+ 14,
Review comment:
Is this the length? If so, we've more than halved the number of values
which can be stored. Is this a good thing? Or, did we force-set the limit
somewhere?
##########
File path: core/src/main/java/org/apache/druid/segment/column/Types.java
##########
@@ -112,4 +121,529 @@
return (typeSignature1 != null && typeSignature1.is(typeDescriptor)) ||
(typeSignature2 != null && typeSignature2.is(typeDescriptor));
}
+
+ /**
+ * Get an {@link ObjectByteStrategy} registered to some {@link
TypeSignature#getComplexTypeName()}.
+ */
+ @Nullable
+ public static ObjectByteStrategy<?> getStrategy(String type)
+ {
+ return STRATEGIES.get(type);
+ }
+
+ /**
+ * hmm... this might look familiar... (see ComplexMetrics)
+ *
+ * Register a complex type name -> {@link ObjectByteStrategy} mapping.
+ *
+ * If the specified type name or type id are already used and the supplied
{@link ObjectByteStrategy} is not of the
+ * same type as the existing value in the map for said key, an {@link ISE}
is thrown.
+ *
+ * @param strategy The {@link ObjectByteStrategy} object to be associated
with the 'type' in the map.
+ */
+ public static void registerStrategy(String typeName, ObjectByteStrategy<?>
strategy)
+ {
+ Preconditions.checkNotNull(typeName);
+ STRATEGIES.compute(typeName, (key, value) -> {
+ if (value == null) {
+ return strategy;
+ } else {
+ if (!value.getClass().getName().equals(strategy.getClass().getName()))
{
+ throw new ISE(
+ "Incompatible strategy for type[%s] already exists. Expected
[%s], found [%s].",
+ key,
+ strategy.getClass().getName(),
+ value.getClass().getName()
+ );
+ } else {
+ return value;
+ }
+ }
+ });
+ }
+
+ /**
+ * Clear and set the 'null' byte of a nullable value to {@link
NullHandling#IS_NULL_BYTE} to a {@link ByteBuffer} at
+ * the supplied position. This method does not change the buffer position,
limit, or mark.
+ *
+ * Nullable types are stored with a leading byte to indicate if the value is
null, followed by teh value bytes
+ * (if not null)
+ *
+ * layout: | null (byte) | value |
+ *
+ * @return number of bytes written (always 1)
+ */
+ public static int writeNull(ByteBuffer buffer, int offset)
+ {
+ buffer.put(offset, NullHandling.IS_NULL_BYTE);
+ return 1;
+ }
+
+ /**
+ * Checks if a 'nullable' value's null byte is set to {@link
NullHandling#IS_NULL_BYTE}. This method will mask the
+ * value of the null byte to only check if the null bit is set, meaning that
the higher bits can be utilized for
+ * flags as necessary (e.g. using high bits to indicate if the value has
been set or not for aggregators).
+ *
+ * Note that writing nullable values with the methods of {@link Types} will
always clear and set the null byte to
+ * either {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE}, losing any flag bits.
+ *
+ * layout: | null (byte) | value |
+ */
+ public static boolean isNullableNull(ByteBuffer buffer, int offset)
+ {
+ // use & so that callers can use the high bits of the null byte to pack
additional information if necessary
+ return (buffer.get(offset) & NullHandling.IS_NULL_BYTE) ==
NullHandling.IS_NULL_BYTE;
+ }
+
+ /**
+ * Write a non-null long value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the long value is written
in the next 8 bytes.
+ *
+ * layout: | null (byte) | long |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 9)
+ */
+ public static int writeNullableLong(ByteBuffer buffer, int offset, long
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putLong(offset, value);
+ return NULLABLE_LONG_SIZE;
+ }
+
+ /**
+ * Reads a non-null long value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | long |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static long readNullableLong(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getLong(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a non-null double value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the double value is written
in the next 8 bytes.
+ *
+ * layout: | null (byte) | double |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 9)
+ */
+ public static int writeNullableDouble(ByteBuffer buffer, int offset, double
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putDouble(offset, value);
+ return NULLABLE_DOUBLE_SIZE;
+ }
+
+ /**
+ * Reads a non-null double value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | double |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static double readNullableDouble(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getDouble(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a non-null float value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the float value is written
in the next 4 bytes.
+ *
+ * layout: | null (byte) | float |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 5)
+ */
+ public static int writeNullableFloat(ByteBuffer buffer, int offset, float
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putFloat(offset, value);
+ return NULLABLE_FLOAT_SIZE;
+ }
+
+ /**
+ * Reads a non-null float value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | float |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static float readNullableFloat(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getFloat(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a variably lengthed byte[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the byte[] value
+ * is not null, the size in bytes is written as an integer in the next 4
bytes, followed by the byte[] value itself.
+ *
+ * layout: | null (byte) | size (int) | byte[] |
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of byte[] if not)
+ */
+ public static int writeNullableVariableBlob(ByteBuffer buffer, int offset,
@Nullable byte[] value)
+ {
+ // | null (byte) | length (int) | bytes |
+ final int size;
+ if (value != null) {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putInt(offset, value.length);
+ offset += Integer.BYTES;
+ final int oldPosition = buffer.position();
+ buffer.position(offset);
+ buffer.put(value, 0, value.length);
+ buffer.position(oldPosition);
+ size = 1 + Integer.BYTES + value.length;
+ } else {
+ size = writeNull(buffer, offset);
+ }
+ return size;
+ }
+
+ /**
+ * Reads a nullable variably lengthed byte[] value from a {@link ByteBuffer}
at the supplied offset. If the null byte
+ * is set to {@link NullHandling#IS_NULL_BYTE}, this method will return
null, else it will read the next 4 bytes to
+ * get the byte[] size followed by that many bytes to extract the value.
+ *
+ * layout: | null (byte) | size (int) | byte[] |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ @Nullable
+ public static byte[] readNullableVariableBlob(ByteBuffer buffer, int offset)
+ {
+ // | null (byte) | length (int) | bytes |
+ final int length = buffer.getInt(offset + VALUE_OFFSET);
+ final byte[] blob = new byte[length];
+ final int oldPosition = buffer.position();
+ buffer.position(offset + VALUE_OFFSET + Integer.BYTES);
+ buffer.get(blob, 0, length);
+ buffer.position(oldPosition);
+ return blob;
+ }
+
+ /**
+ * Write a variably lengthed Long[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the Long[] value
+ * is not null, the size in bytes is written as an integer in the next 4
bytes. Elements of the array are each written
+ * out with {@link #writeNull} if null, or {@link #writeNullableLong} if
not, taking either 1 or 9 bytes each. If the
+ * total byte size of serializing the array is larger than the max size
parameter, this method will explode via a call
+ * to {@link #checkMaxBytes}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | long |, | null
(byte) |, ... |null (byte) | long |} |
+ *
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of Long[] if not)
+ */
+ public static int writeNullableLongArray(ByteBuffer buffer, int offset,
@Nullable Long[] array, int maxSizeBytes)
+ {
+ // | null (byte) | array length (int) | array bytes |
+ if (array == null) {
+ return writeNull(buffer, offset);
+ } else {
+ int sizeBytes = 1 + Integer.BYTES;
+
+ buffer.put(offset, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putInt(offset + 1, array.length);
+ for (Long element : array) {
+ if (element != null) {
+ checkMaxBytes(
+ ColumnType.LONG_ARRAY,
+ sizeBytes + 1 + Long.BYTES,
+ maxSizeBytes
+ );
+ sizeBytes += writeNullableLong(buffer, offset + sizeBytes, element);
+ } else {
+ checkMaxBytes(
+ ColumnType.LONG_ARRAY,
+ sizeBytes + 1,
+ maxSizeBytes
+ );
+ sizeBytes += writeNull(buffer, offset + sizeBytes);
+ }
+ }
+ return sizeBytes;
+ }
+ }
+
+ /**
+ * Reads a nullable variably lengthed Long[] value from a {@link ByteBuffer}
at the supplied offset. If the null byte
+ * is set to {@link NullHandling#IS_NULL_BYTE}, this method will return
null, else it will read the size of the array
+ * from the next 4 bytes and then read that many elements with {@link
#isNullableNull} and {@link #readNullableLong}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | long |, | null
(byte) |, ... |null (byte) | long |} |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ @Nullable
+ public static Long[] readNullableLongArray(ByteBuffer buffer, int offset)
+ {
+ // | null (byte) | array length (int) | array bytes |
+ if (isNullableNull(buffer, offset++)) {
+ return null;
+ }
+ final int longArrayLength = buffer.getInt(offset);
+ offset += Integer.BYTES;
+ final Long[] longs = new Long[longArrayLength];
+ for (int i = 0; i < longArrayLength; i++) {
+ if (isNullableNull(buffer, offset)) {
+ longs[i] = null;
+ } else {
+ longs[i] = readNullableLong(buffer, offset);
+ offset += Long.BYTES;
+ }
+ offset++;
+ }
+ return longs;
+ }
+
+ /**
+ * Write a variably lengthed Double[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the Long[] value
+ * is not null, the size in bytes is written as an integer in the next 4
bytes. Elements of the array are each written
+ * out with {@link #writeNull} if null, or {@link #writeNullableDouble} if
not, taking either 1 or 9 bytes each. If
+ * the total byte size of serializing the array is larger than the max size
parameter, this method will explode via a
+ * call to {@link #checkMaxBytes}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | double |, | null
(byte) |, ... |null (byte) | double |} |
+ *
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of Double[] if
not)
+ */
+ public static int writeNullableDoubleArray(ByteBuffer buffer, int offset,
@Nullable Double[] array, int maxSizeBytes)
+ {
+ // | null (byte) | array length (int) | array bytes |
+ if (array == null) {
+ return writeNull(buffer, offset);
+ } else {
Review comment:
Nit: if the then clause returns, the else clause can be omitted and code
outdented one level to indicate that this is the "main event" assuming you get
in the non-null door?
##########
File path: core/src/main/java/org/apache/druid/segment/column/Types.java
##########
@@ -112,4 +121,529 @@
return (typeSignature1 != null && typeSignature1.is(typeDescriptor)) ||
(typeSignature2 != null && typeSignature2.is(typeDescriptor));
}
+
+ /**
+ * Get an {@link ObjectByteStrategy} registered to some {@link
TypeSignature#getComplexTypeName()}.
+ */
+ @Nullable
+ public static ObjectByteStrategy<?> getStrategy(String type)
+ {
+ return STRATEGIES.get(type);
+ }
+
+ /**
+ * hmm... this might look familiar... (see ComplexMetrics)
+ *
+ * Register a complex type name -> {@link ObjectByteStrategy} mapping.
+ *
+ * If the specified type name or type id are already used and the supplied
{@link ObjectByteStrategy} is not of the
+ * same type as the existing value in the map for said key, an {@link ISE}
is thrown.
+ *
+ * @param strategy The {@link ObjectByteStrategy} object to be associated
with the 'type' in the map.
+ */
+ public static void registerStrategy(String typeName, ObjectByteStrategy<?>
strategy)
+ {
+ Preconditions.checkNotNull(typeName);
+ STRATEGIES.compute(typeName, (key, value) -> {
+ if (value == null) {
+ return strategy;
+ } else {
+ if (!value.getClass().getName().equals(strategy.getClass().getName()))
{
+ throw new ISE(
+ "Incompatible strategy for type[%s] already exists. Expected
[%s], found [%s].",
+ key,
+ strategy.getClass().getName(),
+ value.getClass().getName()
+ );
+ } else {
+ return value;
+ }
+ }
+ });
+ }
+
+ /**
+ * Clear and set the 'null' byte of a nullable value to {@link
NullHandling#IS_NULL_BYTE} to a {@link ByteBuffer} at
+ * the supplied position. This method does not change the buffer position,
limit, or mark.
+ *
+ * Nullable types are stored with a leading byte to indicate if the value is
null, followed by teh value bytes
+ * (if not null)
+ *
+ * layout: | null (byte) | value |
+ *
+ * @return number of bytes written (always 1)
+ */
+ public static int writeNull(ByteBuffer buffer, int offset)
+ {
+ buffer.put(offset, NullHandling.IS_NULL_BYTE);
+ return 1;
+ }
+
+ /**
+ * Checks if a 'nullable' value's null byte is set to {@link
NullHandling#IS_NULL_BYTE}. This method will mask the
+ * value of the null byte to only check if the null bit is set, meaning that
the higher bits can be utilized for
+ * flags as necessary (e.g. using high bits to indicate if the value has
been set or not for aggregators).
+ *
+ * Note that writing nullable values with the methods of {@link Types} will
always clear and set the null byte to
+ * either {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE}, losing any flag bits.
+ *
+ * layout: | null (byte) | value |
+ */
+ public static boolean isNullableNull(ByteBuffer buffer, int offset)
+ {
+ // use & so that callers can use the high bits of the null byte to pack
additional information if necessary
+ return (buffer.get(offset) & NullHandling.IS_NULL_BYTE) ==
NullHandling.IS_NULL_BYTE;
+ }
+
+ /**
+ * Write a non-null long value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the long value is written
in the next 8 bytes.
+ *
+ * layout: | null (byte) | long |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 9)
+ */
+ public static int writeNullableLong(ByteBuffer buffer, int offset, long
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putLong(offset, value);
+ return NULLABLE_LONG_SIZE;
+ }
+
+ /**
+ * Reads a non-null long value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | long |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static long readNullableLong(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getLong(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a non-null double value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the double value is written
in the next 8 bytes.
+ *
+ * layout: | null (byte) | double |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 9)
+ */
+ public static int writeNullableDouble(ByteBuffer buffer, int offset, double
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putDouble(offset, value);
+ return NULLABLE_DOUBLE_SIZE;
+ }
+
+ /**
+ * Reads a non-null double value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | double |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static double readNullableDouble(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getDouble(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a non-null float value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the float value is written
in the next 4 bytes.
+ *
+ * layout: | null (byte) | float |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 5)
+ */
+ public static int writeNullableFloat(ByteBuffer buffer, int offset, float
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putFloat(offset, value);
+ return NULLABLE_FLOAT_SIZE;
+ }
+
+ /**
+ * Reads a non-null float value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | float |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static float readNullableFloat(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getFloat(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a variably lengthed byte[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the byte[] value
+ * is not null, the size in bytes is written as an integer in the next 4
bytes, followed by the byte[] value itself.
+ *
+ * layout: | null (byte) | size (int) | byte[] |
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of byte[] if not)
+ */
+ public static int writeNullableVariableBlob(ByteBuffer buffer, int offset,
@Nullable byte[] value)
+ {
+ // | null (byte) | length (int) | bytes |
+ final int size;
+ if (value != null) {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putInt(offset, value.length);
+ offset += Integer.BYTES;
+ final int oldPosition = buffer.position();
+ buffer.position(offset);
+ buffer.put(value, 0, value.length);
+ buffer.position(oldPosition);
+ size = 1 + Integer.BYTES + value.length;
+ } else {
+ size = writeNull(buffer, offset);
+ }
+ return size;
+ }
+
+ /**
+ * Reads a nullable variably lengthed byte[] value from a {@link ByteBuffer}
at the supplied offset. If the null byte
+ * is set to {@link NullHandling#IS_NULL_BYTE}, this method will return
null, else it will read the next 4 bytes to
+ * get the byte[] size followed by that many bytes to extract the value.
+ *
+ * layout: | null (byte) | size (int) | byte[] |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ @Nullable
+ public static byte[] readNullableVariableBlob(ByteBuffer buffer, int offset)
+ {
+ // | null (byte) | length (int) | bytes |
+ final int length = buffer.getInt(offset + VALUE_OFFSET);
+ final byte[] blob = new byte[length];
+ final int oldPosition = buffer.position();
+ buffer.position(offset + VALUE_OFFSET + Integer.BYTES);
+ buffer.get(blob, 0, length);
+ buffer.position(oldPosition);
+ return blob;
+ }
+
+ /**
+ * Write a variably lengthed Long[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the Long[] value
+ * is not null, the size in bytes is written as an integer in the next 4
bytes. Elements of the array are each written
+ * out with {@link #writeNull} if null, or {@link #writeNullableLong} if
not, taking either 1 or 9 bytes each. If the
+ * total byte size of serializing the array is larger than the max size
parameter, this method will explode via a call
+ * to {@link #checkMaxBytes}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | long |, | null
(byte) |, ... |null (byte) | long |} |
+ *
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of Long[] if not)
+ */
+ public static int writeNullableLongArray(ByteBuffer buffer, int offset,
@Nullable Long[] array, int maxSizeBytes)
+ {
+ // | null (byte) | array length (int) | array bytes |
+ if (array == null) {
+ return writeNull(buffer, offset);
+ } else {
+ int sizeBytes = 1 + Integer.BYTES;
+
+ buffer.put(offset, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putInt(offset + 1, array.length);
+ for (Long element : array) {
+ if (element != null) {
+ checkMaxBytes(
+ ColumnType.LONG_ARRAY,
+ sizeBytes + 1 + Long.BYTES,
+ maxSizeBytes
+ );
+ sizeBytes += writeNullableLong(buffer, offset + sizeBytes, element);
+ } else {
+ checkMaxBytes(
+ ColumnType.LONG_ARRAY,
+ sizeBytes + 1,
+ maxSizeBytes
+ );
+ sizeBytes += writeNull(buffer, offset + sizeBytes);
+ }
+ }
+ return sizeBytes;
+ }
+ }
+
+ /**
+ * Reads a nullable variably lengthed Long[] value from a {@link ByteBuffer}
at the supplied offset. If the null byte
+ * is set to {@link NullHandling#IS_NULL_BYTE}, this method will return
null, else it will read the size of the array
+ * from the next 4 bytes and then read that many elements with {@link
#isNullableNull} and {@link #readNullableLong}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | long |, | null
(byte) |, ... |null (byte) | long |} |
+ *
+ * This method does not change the buffer position, limit, or mark.
Review comment:
Why? The byte buffer has a nice mechanism to keep track of the write
mechanism. These functions recreate that by taking a write position, computing
the amount of data written, returning that value, and asking the caller to
compute the new write offset. These functions write variable amounts of data,
so that there is only one right place to write the next value: after the
current one.
Maybe explain why we need to recreate the byte buffer write pointer?
##########
File path: core/src/main/java/org/apache/druid/segment/column/Types.java
##########
@@ -112,4 +121,529 @@
return (typeSignature1 != null && typeSignature1.is(typeDescriptor)) ||
(typeSignature2 != null && typeSignature2.is(typeDescriptor));
}
+
+ /**
+ * Get an {@link ObjectByteStrategy} registered to some {@link
TypeSignature#getComplexTypeName()}.
+ */
+ @Nullable
+ public static ObjectByteStrategy<?> getStrategy(String type)
+ {
+ return STRATEGIES.get(type);
+ }
+
+ /**
+ * hmm... this might look familiar... (see ComplexMetrics)
+ *
+ * Register a complex type name -> {@link ObjectByteStrategy} mapping.
+ *
+ * If the specified type name or type id are already used and the supplied
{@link ObjectByteStrategy} is not of the
+ * same type as the existing value in the map for said key, an {@link ISE}
is thrown.
+ *
+ * @param strategy The {@link ObjectByteStrategy} object to be associated
with the 'type' in the map.
+ */
+ public static void registerStrategy(String typeName, ObjectByteStrategy<?>
strategy)
+ {
+ Preconditions.checkNotNull(typeName);
+ STRATEGIES.compute(typeName, (key, value) -> {
+ if (value == null) {
+ return strategy;
+ } else {
+ if (!value.getClass().getName().equals(strategy.getClass().getName()))
{
+ throw new ISE(
+ "Incompatible strategy for type[%s] already exists. Expected
[%s], found [%s].",
+ key,
+ strategy.getClass().getName(),
+ value.getClass().getName()
+ );
+ } else {
+ return value;
+ }
+ }
+ });
+ }
+
+ /**
+ * Clear and set the 'null' byte of a nullable value to {@link
NullHandling#IS_NULL_BYTE} to a {@link ByteBuffer} at
+ * the supplied position. This method does not change the buffer position,
limit, or mark.
+ *
+ * Nullable types are stored with a leading byte to indicate if the value is
null, followed by teh value bytes
+ * (if not null)
+ *
+ * layout: | null (byte) | value |
+ *
+ * @return number of bytes written (always 1)
+ */
+ public static int writeNull(ByteBuffer buffer, int offset)
+ {
+ buffer.put(offset, NullHandling.IS_NULL_BYTE);
+ return 1;
+ }
+
+ /**
+ * Checks if a 'nullable' value's null byte is set to {@link
NullHandling#IS_NULL_BYTE}. This method will mask the
+ * value of the null byte to only check if the null bit is set, meaning that
the higher bits can be utilized for
+ * flags as necessary (e.g. using high bits to indicate if the value has
been set or not for aggregators).
+ *
+ * Note that writing nullable values with the methods of {@link Types} will
always clear and set the null byte to
+ * either {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE}, losing any flag bits.
+ *
+ * layout: | null (byte) | value |
+ */
+ public static boolean isNullableNull(ByteBuffer buffer, int offset)
+ {
+ // use & so that callers can use the high bits of the null byte to pack
additional information if necessary
+ return (buffer.get(offset) & NullHandling.IS_NULL_BYTE) ==
NullHandling.IS_NULL_BYTE;
+ }
+
+ /**
+ * Write a non-null long value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the long value is written
in the next 8 bytes.
+ *
+ * layout: | null (byte) | long |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 9)
+ */
+ public static int writeNullableLong(ByteBuffer buffer, int offset, long
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putLong(offset, value);
+ return NULLABLE_LONG_SIZE;
+ }
+
+ /**
+ * Reads a non-null long value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | long |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static long readNullableLong(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getLong(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a non-null double value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the double value is written
in the next 8 bytes.
+ *
+ * layout: | null (byte) | double |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 9)
+ */
+ public static int writeNullableDouble(ByteBuffer buffer, int offset, double
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putDouble(offset, value);
+ return NULLABLE_DOUBLE_SIZE;
+ }
+
+ /**
+ * Reads a non-null double value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | double |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static double readNullableDouble(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getDouble(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a non-null float value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the float value is written
in the next 4 bytes.
+ *
+ * layout: | null (byte) | float |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 5)
+ */
+ public static int writeNullableFloat(ByteBuffer buffer, int offset, float
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putFloat(offset, value);
+ return NULLABLE_FLOAT_SIZE;
+ }
+
+ /**
+ * Reads a non-null float value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | float |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static float readNullableFloat(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getFloat(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a variably lengthed byte[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the byte[] value
+ * is not null, the size in bytes is written as an integer in the next 4
bytes, followed by the byte[] value itself.
+ *
+ * layout: | null (byte) | size (int) | byte[] |
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of byte[] if not)
+ */
+ public static int writeNullableVariableBlob(ByteBuffer buffer, int offset,
@Nullable byte[] value)
+ {
+ // | null (byte) | length (int) | bytes |
+ final int size;
+ if (value != null) {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putInt(offset, value.length);
+ offset += Integer.BYTES;
+ final int oldPosition = buffer.position();
+ buffer.position(offset);
+ buffer.put(value, 0, value.length);
+ buffer.position(oldPosition);
+ size = 1 + Integer.BYTES + value.length;
+ } else {
+ size = writeNull(buffer, offset);
+ }
+ return size;
+ }
+
+ /**
+ * Reads a nullable variably lengthed byte[] value from a {@link ByteBuffer}
at the supplied offset. If the null byte
+ * is set to {@link NullHandling#IS_NULL_BYTE}, this method will return
null, else it will read the next 4 bytes to
+ * get the byte[] size followed by that many bytes to extract the value.
+ *
+ * layout: | null (byte) | size (int) | byte[] |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ @Nullable
+ public static byte[] readNullableVariableBlob(ByteBuffer buffer, int offset)
+ {
+ // | null (byte) | length (int) | bytes |
+ final int length = buffer.getInt(offset + VALUE_OFFSET);
+ final byte[] blob = new byte[length];
+ final int oldPosition = buffer.position();
+ buffer.position(offset + VALUE_OFFSET + Integer.BYTES);
+ buffer.get(blob, 0, length);
+ buffer.position(oldPosition);
+ return blob;
+ }
+
+ /**
+ * Write a variably lengthed Long[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the Long[] value
+ * is not null, the size in bytes is written as an integer in the next 4
bytes. Elements of the array are each written
+ * out with {@link #writeNull} if null, or {@link #writeNullableLong} if
not, taking either 1 or 9 bytes each. If the
+ * total byte size of serializing the array is larger than the max size
parameter, this method will explode via a call
+ * to {@link #checkMaxBytes}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | long |, | null
(byte) |, ... |null (byte) | long |} |
+ *
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of Long[] if not)
+ */
+ public static int writeNullableLongArray(ByteBuffer buffer, int offset,
@Nullable Long[] array, int maxSizeBytes)
+ {
+ // | null (byte) | array length (int) | array bytes |
+ if (array == null) {
+ return writeNull(buffer, offset);
+ } else {
+ int sizeBytes = 1 + Integer.BYTES;
+
+ buffer.put(offset, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putInt(offset + 1, array.length);
+ for (Long element : array) {
+ if (element != null) {
+ checkMaxBytes(
+ ColumnType.LONG_ARRAY,
+ sizeBytes + 1 + Long.BYTES,
+ maxSizeBytes
+ );
+ sizeBytes += writeNullableLong(buffer, offset + sizeBytes, element);
+ } else {
+ checkMaxBytes(
+ ColumnType.LONG_ARRAY,
+ sizeBytes + 1,
+ maxSizeBytes
+ );
+ sizeBytes += writeNull(buffer, offset + sizeBytes);
+ }
+ }
+ return sizeBytes;
+ }
+ }
+
+ /**
+ * Reads a nullable variably lengthed Long[] value from a {@link ByteBuffer}
at the supplied offset. If the null byte
+ * is set to {@link NullHandling#IS_NULL_BYTE}, this method will return
null, else it will read the size of the array
+ * from the next 4 bytes and then read that many elements with {@link
#isNullableNull} and {@link #readNullableLong}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | long |, | null
(byte) |, ... |null (byte) | long |} |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ @Nullable
+ public static Long[] readNullableLongArray(ByteBuffer buffer, int offset)
+ {
+ // | null (byte) | array length (int) | array bytes |
+ if (isNullableNull(buffer, offset++)) {
+ return null;
+ }
+ final int longArrayLength = buffer.getInt(offset);
+ offset += Integer.BYTES;
+ final Long[] longs = new Long[longArrayLength];
+ for (int i = 0; i < longArrayLength; i++) {
+ if (isNullableNull(buffer, offset)) {
+ longs[i] = null;
+ } else {
+ longs[i] = readNullableLong(buffer, offset);
+ offset += Long.BYTES;
+ }
+ offset++;
+ }
+ return longs;
+ }
+
+ /**
+ * Write a variably lengthed Double[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the Long[] value
+ * is not null, the size in bytes is written as an integer in the next 4
bytes. Elements of the array are each written
+ * out with {@link #writeNull} if null, or {@link #writeNullableDouble} if
not, taking either 1 or 9 bytes each. If
+ * the total byte size of serializing the array is larger than the max size
parameter, this method will explode via a
+ * call to {@link #checkMaxBytes}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | double |, | null
(byte) |, ... |null (byte) | double |} |
+ *
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of Double[] if
not)
+ */
+ public static int writeNullableDoubleArray(ByteBuffer buffer, int offset,
@Nullable Double[] array, int maxSizeBytes)
+ {
+ // | null (byte) | array length (int) | array bytes |
+ if (array == null) {
+ return writeNull(buffer, offset);
+ } else {
+ int sizeBytes = 1 + Integer.BYTES;
+ buffer.put(offset, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putInt(offset + 1, array.length);
+ for (Double element : array) {
+ if (element != null) {
Review comment:
Nit: since we handle both null and non-null cases, the logic is a bit
easier to read if we have the `if` be `element == null`. Thus, "if null do this
else do that" rather than "if not null do this else do that".
##########
File path: core/src/main/java/org/apache/druid/segment/column/Types.java
##########
@@ -112,4 +121,529 @@
return (typeSignature1 != null && typeSignature1.is(typeDescriptor)) ||
(typeSignature2 != null && typeSignature2.is(typeDescriptor));
}
+
+ /**
+ * Get an {@link ObjectByteStrategy} registered to some {@link
TypeSignature#getComplexTypeName()}.
+ */
+ @Nullable
+ public static ObjectByteStrategy<?> getStrategy(String type)
+ {
+ return STRATEGIES.get(type);
+ }
+
+ /**
+ * hmm... this might look familiar... (see ComplexMetrics)
+ *
+ * Register a complex type name -> {@link ObjectByteStrategy} mapping.
+ *
+ * If the specified type name or type id are already used and the supplied
{@link ObjectByteStrategy} is not of the
+ * same type as the existing value in the map for said key, an {@link ISE}
is thrown.
+ *
+ * @param strategy The {@link ObjectByteStrategy} object to be associated
with the 'type' in the map.
+ */
+ public static void registerStrategy(String typeName, ObjectByteStrategy<?>
strategy)
+ {
+ Preconditions.checkNotNull(typeName);
+ STRATEGIES.compute(typeName, (key, value) -> {
+ if (value == null) {
+ return strategy;
+ } else {
+ if (!value.getClass().getName().equals(strategy.getClass().getName()))
{
+ throw new ISE(
+ "Incompatible strategy for type[%s] already exists. Expected
[%s], found [%s].",
+ key,
+ strategy.getClass().getName(),
+ value.getClass().getName()
+ );
+ } else {
+ return value;
+ }
+ }
+ });
+ }
+
+ /**
+ * Clear and set the 'null' byte of a nullable value to {@link
NullHandling#IS_NULL_BYTE} to a {@link ByteBuffer} at
+ * the supplied position. This method does not change the buffer position,
limit, or mark.
+ *
+ * Nullable types are stored with a leading byte to indicate if the value is
null, followed by teh value bytes
+ * (if not null)
+ *
+ * layout: | null (byte) | value |
+ *
+ * @return number of bytes written (always 1)
+ */
+ public static int writeNull(ByteBuffer buffer, int offset)
+ {
+ buffer.put(offset, NullHandling.IS_NULL_BYTE);
+ return 1;
+ }
+
+ /**
+ * Checks if a 'nullable' value's null byte is set to {@link
NullHandling#IS_NULL_BYTE}. This method will mask the
+ * value of the null byte to only check if the null bit is set, meaning that
the higher bits can be utilized for
+ * flags as necessary (e.g. using high bits to indicate if the value has
been set or not for aggregators).
+ *
+ * Note that writing nullable values with the methods of {@link Types} will
always clear and set the null byte to
+ * either {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE}, losing any flag bits.
+ *
+ * layout: | null (byte) | value |
+ */
+ public static boolean isNullableNull(ByteBuffer buffer, int offset)
+ {
+ // use & so that callers can use the high bits of the null byte to pack
additional information if necessary
+ return (buffer.get(offset) & NullHandling.IS_NULL_BYTE) ==
NullHandling.IS_NULL_BYTE;
+ }
+
+ /**
+ * Write a non-null long value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the long value is written
in the next 8 bytes.
+ *
+ * layout: | null (byte) | long |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 9)
+ */
+ public static int writeNullableLong(ByteBuffer buffer, int offset, long
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putLong(offset, value);
+ return NULLABLE_LONG_SIZE;
+ }
+
+ /**
+ * Reads a non-null long value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | long |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static long readNullableLong(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getLong(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a non-null double value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the double value is written
in the next 8 bytes.
+ *
+ * layout: | null (byte) | double |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 9)
+ */
+ public static int writeNullableDouble(ByteBuffer buffer, int offset, double
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putDouble(offset, value);
+ return NULLABLE_DOUBLE_SIZE;
+ }
+
+ /**
+ * Reads a non-null double value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | double |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static double readNullableDouble(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getDouble(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a non-null float value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the float value is written
in the next 4 bytes.
+ *
+ * layout: | null (byte) | float |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 5)
+ */
+ public static int writeNullableFloat(ByteBuffer buffer, int offset, float
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putFloat(offset, value);
+ return NULLABLE_FLOAT_SIZE;
+ }
+
+ /**
+ * Reads a non-null float value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | float |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static float readNullableFloat(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getFloat(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a variably lengthed byte[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the byte[] value
+ * is not null, the size in bytes is written as an integer in the next 4
bytes, followed by the byte[] value itself.
+ *
+ * layout: | null (byte) | size (int) | byte[] |
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of byte[] if not)
+ */
+ public static int writeNullableVariableBlob(ByteBuffer buffer, int offset,
@Nullable byte[] value)
+ {
+ // | null (byte) | length (int) | bytes |
+ final int size;
+ if (value != null) {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putInt(offset, value.length);
+ offset += Integer.BYTES;
+ final int oldPosition = buffer.position();
+ buffer.position(offset);
+ buffer.put(value, 0, value.length);
+ buffer.position(oldPosition);
+ size = 1 + Integer.BYTES + value.length;
+ } else {
+ size = writeNull(buffer, offset);
+ }
+ return size;
+ }
+
+ /**
+ * Reads a nullable variably lengthed byte[] value from a {@link ByteBuffer}
at the supplied offset. If the null byte
+ * is set to {@link NullHandling#IS_NULL_BYTE}, this method will return
null, else it will read the next 4 bytes to
+ * get the byte[] size followed by that many bytes to extract the value.
+ *
+ * layout: | null (byte) | size (int) | byte[] |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ @Nullable
+ public static byte[] readNullableVariableBlob(ByteBuffer buffer, int offset)
+ {
+ // | null (byte) | length (int) | bytes |
+ final int length = buffer.getInt(offset + VALUE_OFFSET);
+ final byte[] blob = new byte[length];
+ final int oldPosition = buffer.position();
+ buffer.position(offset + VALUE_OFFSET + Integer.BYTES);
+ buffer.get(blob, 0, length);
+ buffer.position(oldPosition);
+ return blob;
+ }
+
+ /**
+ * Write a variably lengthed Long[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the Long[] value
+ * is not null, the size in bytes is written as an integer in the next 4
bytes. Elements of the array are each written
+ * out with {@link #writeNull} if null, or {@link #writeNullableLong} if
not, taking either 1 or 9 bytes each. If the
+ * total byte size of serializing the array is larger than the max size
parameter, this method will explode via a call
+ * to {@link #checkMaxBytes}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | long |, | null
(byte) |, ... |null (byte) | long |} |
+ *
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of Long[] if not)
+ */
+ public static int writeNullableLongArray(ByteBuffer buffer, int offset,
@Nullable Long[] array, int maxSizeBytes)
+ {
+ // | null (byte) | array length (int) | array bytes |
+ if (array == null) {
+ return writeNull(buffer, offset);
+ } else {
+ int sizeBytes = 1 + Integer.BYTES;
+
+ buffer.put(offset, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putInt(offset + 1, array.length);
+ for (Long element : array) {
+ if (element != null) {
+ checkMaxBytes(
+ ColumnType.LONG_ARRAY,
+ sizeBytes + 1 + Long.BYTES,
+ maxSizeBytes
+ );
+ sizeBytes += writeNullableLong(buffer, offset + sizeBytes, element);
+ } else {
+ checkMaxBytes(
+ ColumnType.LONG_ARRAY,
+ sizeBytes + 1,
+ maxSizeBytes
+ );
+ sizeBytes += writeNull(buffer, offset + sizeBytes);
+ }
+ }
+ return sizeBytes;
+ }
+ }
+
+ /**
+ * Reads a nullable variably lengthed Long[] value from a {@link ByteBuffer}
at the supplied offset. If the null byte
+ * is set to {@link NullHandling#IS_NULL_BYTE}, this method will return
null, else it will read the size of the array
+ * from the next 4 bytes and then read that many elements with {@link
#isNullableNull} and {@link #readNullableLong}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | long |, | null
(byte) |, ... |null (byte) | long |} |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ @Nullable
+ public static Long[] readNullableLongArray(ByteBuffer buffer, int offset)
+ {
+ // | null (byte) | array length (int) | array bytes |
+ if (isNullableNull(buffer, offset++)) {
+ return null;
+ }
+ final int longArrayLength = buffer.getInt(offset);
+ offset += Integer.BYTES;
+ final Long[] longs = new Long[longArrayLength];
+ for (int i = 0; i < longArrayLength; i++) {
+ if (isNullableNull(buffer, offset)) {
+ longs[i] = null;
+ } else {
+ longs[i] = readNullableLong(buffer, offset);
+ offset += Long.BYTES;
+ }
+ offset++;
+ }
+ return longs;
+ }
+
+ /**
+ * Write a variably lengthed Double[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the Long[] value
+ * is not null, the size in bytes is written as an integer in the next 4
bytes. Elements of the array are each written
+ * out with {@link #writeNull} if null, or {@link #writeNullableDouble} if
not, taking either 1 or 9 bytes each. If
+ * the total byte size of serializing the array is larger than the max size
parameter, this method will explode via a
+ * call to {@link #checkMaxBytes}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | double |, | null
(byte) |, ... |null (byte) | double |} |
+ *
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of Double[] if
not)
+ */
+ public static int writeNullableDoubleArray(ByteBuffer buffer, int offset,
@Nullable Double[] array, int maxSizeBytes)
Review comment:
Parachuting into the middle of implementation. How does the max bytes
related to buffer capacity? And, since the array length is a good predictor of
space used, should we enforce an array length (which the user might understand)
vs. a byte length (which seems arbitrary to the user since it is unrelated to
array length or buffer size)?
##########
File path: core/src/main/java/org/apache/druid/segment/column/Types.java
##########
@@ -112,4 +121,529 @@
return (typeSignature1 != null && typeSignature1.is(typeDescriptor)) ||
(typeSignature2 != null && typeSignature2.is(typeDescriptor));
}
+
+ /**
+ * Get an {@link ObjectByteStrategy} registered to some {@link
TypeSignature#getComplexTypeName()}.
+ */
+ @Nullable
+ public static ObjectByteStrategy<?> getStrategy(String type)
+ {
+ return STRATEGIES.get(type);
+ }
+
+ /**
+ * hmm... this might look familiar... (see ComplexMetrics)
+ *
+ * Register a complex type name -> {@link ObjectByteStrategy} mapping.
+ *
+ * If the specified type name or type id are already used and the supplied
{@link ObjectByteStrategy} is not of the
+ * same type as the existing value in the map for said key, an {@link ISE}
is thrown.
+ *
+ * @param strategy The {@link ObjectByteStrategy} object to be associated
with the 'type' in the map.
+ */
+ public static void registerStrategy(String typeName, ObjectByteStrategy<?>
strategy)
+ {
+ Preconditions.checkNotNull(typeName);
+ STRATEGIES.compute(typeName, (key, value) -> {
+ if (value == null) {
+ return strategy;
+ } else {
+ if (!value.getClass().getName().equals(strategy.getClass().getName()))
{
+ throw new ISE(
+ "Incompatible strategy for type[%s] already exists. Expected
[%s], found [%s].",
+ key,
+ strategy.getClass().getName(),
+ value.getClass().getName()
+ );
+ } else {
+ return value;
+ }
+ }
+ });
+ }
+
+ /**
+ * Clear and set the 'null' byte of a nullable value to {@link
NullHandling#IS_NULL_BYTE} to a {@link ByteBuffer} at
+ * the supplied position. This method does not change the buffer position,
limit, or mark.
+ *
+ * Nullable types are stored with a leading byte to indicate if the value is
null, followed by teh value bytes
+ * (if not null)
+ *
+ * layout: | null (byte) | value |
+ *
+ * @return number of bytes written (always 1)
+ */
+ public static int writeNull(ByteBuffer buffer, int offset)
+ {
+ buffer.put(offset, NullHandling.IS_NULL_BYTE);
+ return 1;
+ }
+
+ /**
+ * Checks if a 'nullable' value's null byte is set to {@link
NullHandling#IS_NULL_BYTE}. This method will mask the
+ * value of the null byte to only check if the null bit is set, meaning that
the higher bits can be utilized for
+ * flags as necessary (e.g. using high bits to indicate if the value has
been set or not for aggregators).
+ *
+ * Note that writing nullable values with the methods of {@link Types} will
always clear and set the null byte to
+ * either {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE}, losing any flag bits.
+ *
+ * layout: | null (byte) | value |
+ */
+ public static boolean isNullableNull(ByteBuffer buffer, int offset)
+ {
+ // use & so that callers can use the high bits of the null byte to pack
additional information if necessary
+ return (buffer.get(offset) & NullHandling.IS_NULL_BYTE) ==
NullHandling.IS_NULL_BYTE;
+ }
+
+ /**
+ * Write a non-null long value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the long value is written
in the next 8 bytes.
+ *
+ * layout: | null (byte) | long |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 9)
+ */
+ public static int writeNullableLong(ByteBuffer buffer, int offset, long
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putLong(offset, value);
+ return NULLABLE_LONG_SIZE;
+ }
+
+ /**
+ * Reads a non-null long value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | long |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static long readNullableLong(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getLong(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a non-null double value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the double value is written
in the next 8 bytes.
+ *
+ * layout: | null (byte) | double |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 9)
+ */
+ public static int writeNullableDouble(ByteBuffer buffer, int offset, double
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putDouble(offset, value);
+ return NULLABLE_DOUBLE_SIZE;
+ }
+
+ /**
+ * Reads a non-null double value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | double |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static double readNullableDouble(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getDouble(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a non-null float value to a {@link ByteBuffer} at the supplied
offset. The first byte is always cleared and
+ * set to {@link NullHandling#IS_NOT_NULL_BYTE}, the float value is written
in the next 4 bytes.
+ *
+ * layout: | null (byte) | float |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ *
+ * @return number of bytes written (always 5)
+ */
+ public static int writeNullableFloat(ByteBuffer buffer, int offset, float
value)
+ {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putFloat(offset, value);
+ return NULLABLE_FLOAT_SIZE;
+ }
+
+ /**
+ * Reads a non-null float value from a {@link ByteBuffer} at the supplied
offset. This method should only be called
+ * if and only if {@link #isNullableNull} for the same offset returns false.
+ *
+ * layout: | null (byte) | float |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ public static float readNullableFloat(ByteBuffer buffer, int offset)
+ {
+ assert !isNullableNull(buffer, offset);
+ return buffer.getFloat(offset + VALUE_OFFSET);
+ }
+
+ /**
+ * Write a variably lengthed byte[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the byte[] value
+ * is not null, the size in bytes is written as an integer in the next 4
bytes, followed by the byte[] value itself.
+ *
+ * layout: | null (byte) | size (int) | byte[] |
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of byte[] if not)
+ */
+ public static int writeNullableVariableBlob(ByteBuffer buffer, int offset,
@Nullable byte[] value)
+ {
+ // | null (byte) | length (int) | bytes |
+ final int size;
+ if (value != null) {
+ buffer.put(offset++, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putInt(offset, value.length);
+ offset += Integer.BYTES;
+ final int oldPosition = buffer.position();
+ buffer.position(offset);
+ buffer.put(value, 0, value.length);
+ buffer.position(oldPosition);
+ size = 1 + Integer.BYTES + value.length;
+ } else {
+ size = writeNull(buffer, offset);
+ }
+ return size;
+ }
+
+ /**
+ * Reads a nullable variably lengthed byte[] value from a {@link ByteBuffer}
at the supplied offset. If the null byte
+ * is set to {@link NullHandling#IS_NULL_BYTE}, this method will return
null, else it will read the next 4 bytes to
+ * get the byte[] size followed by that many bytes to extract the value.
+ *
+ * layout: | null (byte) | size (int) | byte[] |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ @Nullable
+ public static byte[] readNullableVariableBlob(ByteBuffer buffer, int offset)
+ {
+ // | null (byte) | length (int) | bytes |
+ final int length = buffer.getInt(offset + VALUE_OFFSET);
+ final byte[] blob = new byte[length];
+ final int oldPosition = buffer.position();
+ buffer.position(offset + VALUE_OFFSET + Integer.BYTES);
+ buffer.get(blob, 0, length);
+ buffer.position(oldPosition);
+ return blob;
+ }
+
+ /**
+ * Write a variably lengthed Long[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the Long[] value
+ * is not null, the size in bytes is written as an integer in the next 4
bytes. Elements of the array are each written
+ * out with {@link #writeNull} if null, or {@link #writeNullableLong} if
not, taking either 1 or 9 bytes each. If the
+ * total byte size of serializing the array is larger than the max size
parameter, this method will explode via a call
+ * to {@link #checkMaxBytes}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | long |, | null
(byte) |, ... |null (byte) | long |} |
+ *
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of Long[] if not)
+ */
+ public static int writeNullableLongArray(ByteBuffer buffer, int offset,
@Nullable Long[] array, int maxSizeBytes)
+ {
+ // | null (byte) | array length (int) | array bytes |
+ if (array == null) {
+ return writeNull(buffer, offset);
+ } else {
+ int sizeBytes = 1 + Integer.BYTES;
+
+ buffer.put(offset, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putInt(offset + 1, array.length);
+ for (Long element : array) {
+ if (element != null) {
+ checkMaxBytes(
+ ColumnType.LONG_ARRAY,
+ sizeBytes + 1 + Long.BYTES,
+ maxSizeBytes
+ );
+ sizeBytes += writeNullableLong(buffer, offset + sizeBytes, element);
+ } else {
+ checkMaxBytes(
+ ColumnType.LONG_ARRAY,
+ sizeBytes + 1,
+ maxSizeBytes
+ );
+ sizeBytes += writeNull(buffer, offset + sizeBytes);
+ }
+ }
+ return sizeBytes;
+ }
+ }
+
+ /**
+ * Reads a nullable variably lengthed Long[] value from a {@link ByteBuffer}
at the supplied offset. If the null byte
+ * is set to {@link NullHandling#IS_NULL_BYTE}, this method will return
null, else it will read the size of the array
+ * from the next 4 bytes and then read that many elements with {@link
#isNullableNull} and {@link #readNullableLong}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | long |, | null
(byte) |, ... |null (byte) | long |} |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ @Nullable
+ public static Long[] readNullableLongArray(ByteBuffer buffer, int offset)
+ {
+ // | null (byte) | array length (int) | array bytes |
+ if (isNullableNull(buffer, offset++)) {
+ return null;
+ }
+ final int longArrayLength = buffer.getInt(offset);
+ offset += Integer.BYTES;
+ final Long[] longs = new Long[longArrayLength];
+ for (int i = 0; i < longArrayLength; i++) {
+ if (isNullableNull(buffer, offset)) {
+ longs[i] = null;
+ } else {
+ longs[i] = readNullableLong(buffer, offset);
+ offset += Long.BYTES;
+ }
+ offset++;
+ }
+ return longs;
+ }
+
+ /**
+ * Write a variably lengthed Double[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the Long[] value
+ * is not null, the size in bytes is written as an integer in the next 4
bytes. Elements of the array are each written
+ * out with {@link #writeNull} if null, or {@link #writeNullableDouble} if
not, taking either 1 or 9 bytes each. If
+ * the total byte size of serializing the array is larger than the max size
parameter, this method will explode via a
+ * call to {@link #checkMaxBytes}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | double |, | null
(byte) |, ... |null (byte) | double |} |
+ *
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of Double[] if
not)
+ */
+ public static int writeNullableDoubleArray(ByteBuffer buffer, int offset,
@Nullable Double[] array, int maxSizeBytes)
+ {
+ // | null (byte) | array length (int) | array bytes |
+ if (array == null) {
+ return writeNull(buffer, offset);
+ } else {
+ int sizeBytes = 1 + Integer.BYTES;
+ buffer.put(offset, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putInt(offset + 1, array.length);
+ for (Double element : array) {
+ if (element != null) {
+ checkMaxBytes(
+ ColumnType.DOUBLE_ARRAY,
+ sizeBytes + 1 + Double.BYTES,
+ maxSizeBytes
+ );
+ sizeBytes += writeNullableDouble(buffer, offset + sizeBytes,
element);
+ } else {
+ checkMaxBytes(
+ ColumnType.DOUBLE_ARRAY,
+ sizeBytes + 1,
+ maxSizeBytes
+ );
+ sizeBytes += writeNull(buffer, offset + sizeBytes);
+ }
+ }
+ return sizeBytes;
+ }
+ }
+
+ /**
+ * Reads a nullable variably lengthed Double[] value from a {@link
ByteBuffer} at the supplied offset. If the null
+ * byte is set to {@link NullHandling#IS_NULL_BYTE}, this method will return
null, else it will read the size of the
+ * array from the next 4 bytes and then read that many elements with {@link
#isNullableNull} and
+ * {@link #readNullableDouble}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | double |, | null
(byte) |, ... |null (byte) | double |} |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ @Nullable
+ public static Double[] readNullableDoubleArray(ByteBuffer buffer, int offset)
+ {
+ // | null (byte) | array length (int) | array bytes |
+ if (isNullableNull(buffer, offset++)) {
+ return null;
+ }
+ final int doubleArrayLength = buffer.getInt(offset);
+ offset += Integer.BYTES;
+ final Double[] doubles = new Double[doubleArrayLength];
+ for (int i = 0; i < doubleArrayLength; i++) {
+ if (isNullableNull(buffer, offset)) {
+ doubles[i] = null;
+ } else {
+ doubles[i] = readNullableDouble(buffer, offset);
+ offset += Double.BYTES;
+ }
+ offset++;
+ }
+ return doubles;
+ }
+
+ /**
+ * Write a variably lengthed String[] value to a {@link ByteBuffer} at the
supplied offset. The first byte is set to
+ * {@link NullHandling#IS_NULL_BYTE} or {@link
NullHandling#IS_NOT_NULL_BYTE} as appropriate, and if the String[]
+ * value is not null, the size in bytes is written as an integer in the next
4 bytes. The Strings themselves are
+ * encoded with {@link StringUtils#toUtf8} Elements of the array are each
written out with {@link #writeNull} if null,
+ * or {@link #writeNullableVariableBlob} if not, taking either 1 or 5 + the
size of the utf8 byte array each. If the
+ * total byte size of serializing the array is larger than the max size
parameter, this method will explode via a
+ * call to {@link #checkMaxBytes}.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | size (int) | byte[]
|, | null (byte) |, ... } |
+ *
+ *
+ * This method does not permanently change the buffer position, limit, or
mark.
+ *
+ * @return number of bytes written (1 if null, or 5 + size of String[] if
not)
+ */
+ public static int writeNullableStringArray(ByteBuffer buffer, int offset,
@Nullable String[] array, int maxSizeBytes)
+ {
+ // | null (byte) | array length (int) | array bytes |
+ if (array == null) {
+ return writeNull(buffer, offset);
+ } else {
+ int sizeBytes = 1 + Integer.BYTES;
+ buffer.put(offset, NullHandling.IS_NOT_NULL_BYTE);
+ buffer.putInt(offset + 1, array.length);
+ for (String element : array) {
+ if (element != null) {
+ final byte[] stringElementBytes = StringUtils.toUtf8(element);
+ checkMaxBytes(
+ ColumnType.STRING_ARRAY,
+ sizeBytes + 1 + Integer.BYTES + stringElementBytes.length,
+ maxSizeBytes
+ );
+ sizeBytes += writeNullableVariableBlob(buffer, offset + sizeBytes,
stringElementBytes);
+ } else {
+ checkMaxBytes(
+ ColumnType.STRING_ARRAY,
+ sizeBytes + 1,
+ maxSizeBytes
+ );
+ sizeBytes += writeNull(buffer, offset + sizeBytes);
+ }
+ }
+ return sizeBytes;
+ }
+ }
+
+ /**
+ * Reads a nullable variably lengthed String[] value from a {@link
ByteBuffer} at the supplied offset. If the null
+ * byte is set to {@link NullHandling#IS_NULL_BYTE}, this method will return
null, else it will read the size of the
+ * array from the next 4 bytes and then read that many elements with {@link
#readNullableVariableBlob} and decode them
+ * with {@link StringUtils#fromUtf8} to convert to string values.
+ *
+ * layout: | null (byte) | size (int) | {| null (byte) | size (int) | byte[]
|, | null (byte) |, ... } |
+ *
+ * This method does not change the buffer position, limit, or mark.
+ */
+ @Nullable
+ public static String[] readNullableStringArray(ByteBuffer buffer, int offset)
+ {
+ // | null (byte) | array length (int) | array bytes |
+ if (isNullableNull(buffer, offset++)) {
+ return null;
+ }
+ final int stringArrayLength = buffer.getInt(offset);
+ offset += Integer.BYTES;
+ final String[] stringArray = new String[stringArrayLength];
+ for (int i = 0; i < stringArrayLength; i++) {
+ if (isNullableNull(buffer, offset)) {
+ stringArray[i] = null;
+ } else {
+ final byte[] stringElementBytes = readNullableVariableBlob(buffer,
offset);
+ stringArray[i] = StringUtils.fromUtf8(stringElementBytes);
+ offset += Integer.BYTES + stringElementBytes.length;
+ }
+ offset++;
Review comment:
Computing the new position is error-prone. Wouldn't it be easier to save
the original position, compute the offset as the final position - the start
position? Code would be simpler and we'd compute sizes once rather than twice.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
