nssalian commented on code in PR #16568:
URL: https://github.com/apache/iceberg/pull/16568#discussion_r3555504226
##########
api/src/main/java/org/apache/iceberg/variants/SerializedArray.java:
##########
@@ -35,30 +35,47 @@ static SerializedArray from(VariantMetadata metadata,
byte[] bytes) {
return from(metadata,
ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN), bytes[0]);
}
+ @VisibleForTesting
static SerializedArray from(VariantMetadata metadata, ByteBuffer value, int
header) {
+ return from(metadata, value, header, 0);
+ }
+
+ static SerializedArray from(VariantMetadata metadata, ByteBuffer value, int
header, int depth) {
Preconditions.checkArgument(
value.order() == ByteOrder.LITTLE_ENDIAN, "Unsupported byte order: big
endian");
BasicType basicType = VariantUtil.basicType(header);
Preconditions.checkArgument(
basicType == BasicType.ARRAY, "Invalid array, basic type: " +
basicType);
- return new SerializedArray(metadata, value, header);
+ return new SerializedArray(metadata, value, header, depth);
}
private final VariantMetadata metadata;
private final ByteBuffer value;
private final int offsetSize;
private final int offsetListOffset;
private final int dataOffset;
+ private final int depth;
private final VariantValue[] array;
- private SerializedArray(VariantMetadata metadata, ByteBuffer value, int
header) {
+ private SerializedArray(VariantMetadata metadata, ByteBuffer value, int
header, int depth) {
this.metadata = metadata;
this.value = value;
+ this.depth = depth;
this.offsetSize = 1 + ((header & OFFSET_SIZE_MASK) >> OFFSET_SIZE_SHIFT);
int numElementsSize = ((header & IS_LARGE) == IS_LARGE) ? 4 : 1;
+ Preconditions.checkArgument(
+ value.remaining() >= HEADER_SIZE + numElementsSize,
+ "Invalid variant array: buffer too small for element count field");
int numElements = ByteBuffers.readLittleEndianUnsigned(value, HEADER_SIZE,
numElementsSize);
+ Preconditions.checkArgument(
+ numElements >= 0, "Invalid variant array: negative element count %s",
numElements);
this.offsetListOffset = HEADER_SIZE + numElementsSize;
- this.dataOffset = offsetListOffset + ((1 + numElements) * offsetSize);
+ long offsetTableEnd = (long) offsetListOffset + ((long) numElements + 1L)
* offsetSize;
+ Preconditions.checkArgument(
+ offsetTableEnd <= value.remaining(),
Review Comment:
Fixed this too. I added an absolute `MAX_ELEMENTS` cap (16,777,216) checked
before the array/object/metadata allocations. parquet-java #3562 has no
equivalent (it bounds by buffer size only), so this one is iceberg-specific and
I've noted that in the javadoc. Added `testExcessiveArrayNumElementsRejected`
plus object and metadata variants.
##########
api/src/main/java/org/apache/iceberg/variants/SerializedMetadata.java:
##########
@@ -59,15 +59,32 @@ static SerializedMetadata from(ByteBuffer metadata) {
private SerializedMetadata(ByteBuffer metadata, int header) {
this.isSorted = (header & SORTED_STRINGS) == SORTED_STRINGS;
this.offsetSize = 1 + ((header & OFFSET_SIZE_MASK) >> OFFSET_SIZE_SHIFT);
+ Preconditions.checkArgument(
+ metadata.remaining() >= HEADER_SIZE + offsetSize,
+ "Invalid variant metadata: buffer too small for dictionary size
field");
int dictSize = ByteBuffers.readLittleEndianUnsigned(metadata, HEADER_SIZE,
offsetSize);
- this.dict = new String[dictSize];
+ Preconditions.checkArgument(
+ dictSize >= 0, "Invalid variant metadata: negative dictionary size
%s", dictSize);
this.offsetListOffset = HEADER_SIZE + offsetSize;
+ long offsetTableEnd = (long) offsetListOffset + ((long) dictSize + 1L) *
offsetSize;
+ Preconditions.checkArgument(
+ offsetTableEnd <= metadata.remaining(),
+ "Invalid variant metadata: dictionary size %s exceeds buffer",
+ dictSize);
+ this.dict = new String[dictSize];
this.dataOffset = offsetListOffset + ((1 + dictSize) * offsetSize);
Review Comment:
It now uses `Math.toIntExact(offsetTableEnd)`, consistent with
`SerializedArray` and `SerializedObject`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
