iemejia commented on code in PR #3864:
URL: https://github.com/apache/avro/pull/3864#discussion_r3568036135
##########
lang/perl/lib/Avro/BinaryDecoder.pm:
##########
@@ -215,27 +387,78 @@ sub decode_enum {
}
sub skip_block {
- my $class = shift;
- my ($reader, $block_content) = @_;
- my $block_count = decode_long($class, undef, undef, $reader);
+ # Called as a plain function by skip_array/skip_map as
+ # skip_block($reader, $min_bytes, $coderef); it is not a method, so there
is
+ # no leading class argument. decode_long ignores its (shifted) first
argument
+ # and reads from the last, so pass __PACKAGE__ for it and $reader last.
+ my ($reader, $min_bytes, $block_content) = @_;
+ # Zero-byte elements loop with no per-item input, so bound them tightly;
+ # other elements consume bytes (bounded by EOF) so only the structural cap
+ # applies. Every collection is capped by the structural limit, so for
+ # zero-byte elements use whichever of the two limits is tighter.
+ my $limit = $MAX_COLLECTION_STRUCTURAL;
+ if ($min_bytes <= 0 && $MAX_COLLECTION_ITEMS < $limit) {
+ $limit = $MAX_COLLECTION_ITEMS;
+ }
+ my $block_count = decode_long(__PACKAGE__, undef, undef, $reader);
+ my $skipped = 0;
while ($block_count) {
if ($block_count < 0) {
- $reader->seek($block_count, 0);
- next;
+ # A byte-sized (negative-count) block still declares an element
+ # count; bound it cumulatively too, so a huge declared count with a
+ # small or zero block size cannot bypass the cap (e.g. zero-byte
+ # elements whose block size is 0).
+ my $count = -$block_count;
+ if ($skipped + $count > $limit) {
+ throw Avro::BinaryDecoder::Error::CollectionSize(
+ "Cannot skip a collection of more than $limit "
+ . "elements (declared @{[ $skipped + $count ]})");
+ }
+ $skipped += $count;
+ # A negative count is followed by a long block size in bytes, which
+ # lets the whole block be skipped without decoding each item. Skip
+ # forward by that many bytes relative to the current position
+ # (SEEK_CUR); whence 0 (SEEK_SET) would seek to an absolute (here
+ # nonsensical) offset. A failed seek is treated as fatal.
+ my $block_size = decode_long(__PACKAGE__, undef, undef, $reader);
+ if ($block_size < 0) {
+ # A negative block size would seek the reader backwards,
+ # risking an infinite loop or mis-decoding of a corrupt input.
+ throw Avro::Schema::Error::Parse(
+ "Invalid negative block size: $block_size");
+ }
+ # Reject a block size that exceeds the bytes actually remaining
+ # before skipping, so a truncated input fails instead of seeking
+ # past EOF.
+ _ensure_available($reader, $block_size, 1);
Review Comment:
Fixed — skip_block's negative-count branch now rejects a block_size too
small to hold the declared count at its minimum on-wire size (`$count >
int($block_size / $min_bytes)` when min_bytes > 0), before seeking.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -84,6 +84,68 @@ EOJ
is $dec, "a", "Binary_Encodings.Complex_Types.Unions-a";
}
+## union and enum index bounds
+{
+ my $union = Avro::Schema->parse(q(["string","null"]));
+ # Index 2 (zig-zag int 0x04) is out of range for a 2-branch union.
+ open my $reader, '<', \"\x04" or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $union,
+ reader_schema => $union,
+ reader => $reader,
+ );
+ } 'Avro::Schema::Error::Parse', "union branch index out of range is
rejected";
+
+ # Index -1 (zig-zag int 0x01) must not wrap to a valid branch.
+ open $reader, '<', \"\x01" or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $union,
+ reader_schema => $union,
+ reader => $reader,
+ );
+ } 'Avro::Schema::Error::Parse', "negative union branch index is rejected";
+
+ my $enum = Avro::Schema->parse(
+ q({ "type": "enum", "name": "e", "symbols": [ "a", "b" ] }));
+ # Index 9 (zig-zag int 0x12) is out of range for a 2-symbol enum.
+ open $reader, '<', \"\x12" or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $enum,
+ reader_schema => $enum,
+ reader => $reader,
+ );
+ } 'Avro::Schema::Error::Parse', "enum symbol index out of range is
rejected";
+
+ # Index -1 (zig-zag int 0x01) must not wrap to a valid symbol.
+ open $reader, '<', \"\x01" or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $enum,
+ reader_schema => $enum,
+ reader => $reader,
+ );
+ } 'Avro::Schema::Error::Parse', "negative enum symbol index is rejected";
+}
+
+## overlong varint
+{
+ # A 64-bit value uses at most 10 bytes; an 11th continuation byte is a
+ # malformed overlong varint and must be rejected.
Review Comment:
Fixed — reworded to 'an 11th byte (10 continuation bytes 0x80 then a
terminator)'.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -255,4 +317,361 @@ EOP
is $dec->{one}[0], 1.0, "kind of dumb test";
}
+## A bytes/string value declares a length prefix; a malicious or truncated
+## input can declare far more bytes than actually exist. On a seekable reader
+## that is rejected before allocating for it.
+{
+ my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" }));
+
+ ## A length prefix declaring 100 MiB, with no payload following it.
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => 100 * 1024 * 1024,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader,
+ );
+ } qr/Cannot read/, "oversized bytes length is rejected before allocating";
+
+ ## A well-formed value above the check threshold whose data is present
+ ## still decodes.
+ my $payload = 'x' x (2 * 1024 * 1024);
+ my $enc2 = '';
+ Avro::BinaryEncoder->encode(
+ schema => $bytes_schema,
+ data => $payload,
+ emit_cb => sub { $enc2 .= ${ $_[0] } },
+ );
+ open my $reader2, '<', \$enc2 or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader2,
+ );
+ is $dec, $payload, "within-limit bytes value still decodes";
+}
+
+## An array/map block declares an element count; a malicious or truncated input
+## can declare far more elements than the remaining bytes could hold. The count
+## is validated against the bytes remaining before iterating, using the minimum
+## on-wire size of the element schema (so 0-byte elements like null are not
+## falsely rejected).
+sub encode_long {
+ my ($value) = @_;
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => $value,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ return $enc;
+}
+
+{
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"long" }));
+ my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized array count is rejected before
iterating";
+}
+
+{
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long"
}));
+ my $enc = encode_long(1_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map count is rejected before
iterating";
+}
+
+{
+ ## An array of nulls: null elements occupy zero bytes, so a large count is
+ ## legitimate and must not be rejected.
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"null" }));
+ my $count = 100_000;
+ my $enc = encode_long($count) . encode_long(0); # one block + end marker
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ is scalar(@$dec), $count, "array of nulls is not falsely rejected";
+}
+
+## Zero-byte elements (null, zero-length fixed, all-null records) consume no
+## input, so the bytes-remaining check cannot bound them. A huge declared block
+## count of such elements is capped instead.
+sub decode_zero_byte_array {
+ my ($items_schema, $enc) = @_;
+ my $schema = Avro::Schema->parse(qq({ "type": "array", "items":
$items_schema }));
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ return Avro::BinaryDecoder->decode(
+ writer_schema => $schema,
+ reader_schema => $schema,
+ reader => $reader,
+ );
+}
+
+{
+ ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is
+ ## rejected by the default limit, before allocating. Pin the limit to the
+ ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS
was
+ ## set in the environment when the module was loaded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS =
+ $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS;
Review Comment:
Addressed globally: rather than localize MAX_COLLECTION_STRUCTURAL per test,
the top of t/03_bin_decode.t now resets BOTH MAX_COLLECTION_ITEMS and
MAX_COLLECTION_STRUCTURAL to their defaults (they're both initialized from
AVRO_MAX_COLLECTION_ITEMS at load). Verified the whole suite passes with
AVRO_MAX_COLLECTION_ITEMS=5 set.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -255,4 +317,361 @@ EOP
is $dec->{one}[0], 1.0, "kind of dumb test";
}
+## A bytes/string value declares a length prefix; a malicious or truncated
+## input can declare far more bytes than actually exist. On a seekable reader
+## that is rejected before allocating for it.
+{
+ my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" }));
+
+ ## A length prefix declaring 100 MiB, with no payload following it.
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => 100 * 1024 * 1024,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader,
+ );
+ } qr/Cannot read/, "oversized bytes length is rejected before allocating";
+
+ ## A well-formed value above the check threshold whose data is present
+ ## still decodes.
+ my $payload = 'x' x (2 * 1024 * 1024);
+ my $enc2 = '';
+ Avro::BinaryEncoder->encode(
+ schema => $bytes_schema,
+ data => $payload,
+ emit_cb => sub { $enc2 .= ${ $_[0] } },
+ );
+ open my $reader2, '<', \$enc2 or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader2,
+ );
+ is $dec, $payload, "within-limit bytes value still decodes";
+}
+
+## An array/map block declares an element count; a malicious or truncated input
+## can declare far more elements than the remaining bytes could hold. The count
+## is validated against the bytes remaining before iterating, using the minimum
+## on-wire size of the element schema (so 0-byte elements like null are not
+## falsely rejected).
+sub encode_long {
+ my ($value) = @_;
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => $value,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ return $enc;
+}
+
+{
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"long" }));
+ my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized array count is rejected before
iterating";
+}
+
+{
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long"
}));
+ my $enc = encode_long(1_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map count is rejected before
iterating";
+}
+
+{
+ ## An array of nulls: null elements occupy zero bytes, so a large count is
+ ## legitimate and must not be rejected.
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"null" }));
+ my $count = 100_000;
+ my $enc = encode_long($count) . encode_long(0); # one block + end marker
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ is scalar(@$dec), $count, "array of nulls is not falsely rejected";
+}
+
+## Zero-byte elements (null, zero-length fixed, all-null records) consume no
+## input, so the bytes-remaining check cannot bound them. A huge declared block
+## count of such elements is capped instead.
+sub decode_zero_byte_array {
+ my ($items_schema, $enc) = @_;
+ my $schema = Avro::Schema->parse(qq({ "type": "array", "items":
$items_schema }));
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ return Avro::BinaryDecoder->decode(
+ writer_schema => $schema,
+ reader_schema => $schema,
+ reader => $reader,
+ );
+}
+
+{
+ ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is
+ ## rejected by the default limit, before allocating. Pin the limit to the
+ ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS
was
+ ## set in the environment when the module was loaded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS =
+ $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(200_000_000) .
encode_long(0));
+ } qr/more than \d+ zero-byte/, "array of 200M nulls rejected by default
limit";
+}
+
+{
+ ## Within a lowered limit, a legitimate small null array still decodes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
Review Comment:
Addressed globally: rather than localize MAX_COLLECTION_STRUCTURAL per test,
the top of t/03_bin_decode.t now resets BOTH MAX_COLLECTION_ITEMS and
MAX_COLLECTION_STRUCTURAL to their defaults (they're both initialized from
AVRO_MAX_COLLECTION_ITEMS at load). Verified the whole suite passes with
AVRO_MAX_COLLECTION_ITEMS=5 set.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -255,4 +317,361 @@ EOP
is $dec->{one}[0], 1.0, "kind of dumb test";
}
+## A bytes/string value declares a length prefix; a malicious or truncated
+## input can declare far more bytes than actually exist. On a seekable reader
+## that is rejected before allocating for it.
+{
+ my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" }));
+
+ ## A length prefix declaring 100 MiB, with no payload following it.
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => 100 * 1024 * 1024,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader,
+ );
+ } qr/Cannot read/, "oversized bytes length is rejected before allocating";
+
+ ## A well-formed value above the check threshold whose data is present
+ ## still decodes.
+ my $payload = 'x' x (2 * 1024 * 1024);
+ my $enc2 = '';
+ Avro::BinaryEncoder->encode(
+ schema => $bytes_schema,
+ data => $payload,
+ emit_cb => sub { $enc2 .= ${ $_[0] } },
+ );
+ open my $reader2, '<', \$enc2 or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader2,
+ );
+ is $dec, $payload, "within-limit bytes value still decodes";
+}
+
+## An array/map block declares an element count; a malicious or truncated input
+## can declare far more elements than the remaining bytes could hold. The count
+## is validated against the bytes remaining before iterating, using the minimum
+## on-wire size of the element schema (so 0-byte elements like null are not
+## falsely rejected).
+sub encode_long {
+ my ($value) = @_;
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => $value,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ return $enc;
+}
+
+{
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"long" }));
+ my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized array count is rejected before
iterating";
+}
+
+{
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long"
}));
+ my $enc = encode_long(1_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map count is rejected before
iterating";
+}
+
+{
+ ## An array of nulls: null elements occupy zero bytes, so a large count is
+ ## legitimate and must not be rejected.
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"null" }));
+ my $count = 100_000;
+ my $enc = encode_long($count) . encode_long(0); # one block + end marker
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ is scalar(@$dec), $count, "array of nulls is not falsely rejected";
+}
+
+## Zero-byte elements (null, zero-length fixed, all-null records) consume no
+## input, so the bytes-remaining check cannot bound them. A huge declared block
+## count of such elements is capped instead.
+sub decode_zero_byte_array {
+ my ($items_schema, $enc) = @_;
+ my $schema = Avro::Schema->parse(qq({ "type": "array", "items":
$items_schema }));
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ return Avro::BinaryDecoder->decode(
+ writer_schema => $schema,
+ reader_schema => $schema,
+ reader => $reader,
+ );
+}
+
+{
+ ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is
+ ## rejected by the default limit, before allocating. Pin the limit to the
+ ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS
was
+ ## set in the environment when the module was loaded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS =
+ $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(200_000_000) .
encode_long(0));
+ } qr/more than \d+ zero-byte/, "array of 200M nulls rejected by default
limit";
+}
+
+{
+ ## Within a lowered limit, a legitimate small null array still decodes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ my $dec = decode_zero_byte_array('"null"', encode_long(1000) .
encode_long(0));
+ is scalar(@$dec), 1000, "array of nulls within configured limit still
reads";
+}
+
+{
+ ## INT64_MIN as a block count is the pathological negation case. Negating
it
+ ## yields 2**63, which the cap rejects. INT64_MIN zig-zag encodes as the
+ ## 10-byte varint below, followed by a block byte-size (0).
+ my $int64_min = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01";
+ throws_ok {
+ decode_zero_byte_array('"null"', $int64_min . encode_long(0));
+ } qr/Cannot read a collection|zero-byte|Invalid/, "INT64_MIN array block
count rejected";
+}
+
+{
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
Review Comment:
Addressed globally: rather than localize MAX_COLLECTION_STRUCTURAL per test,
the top of t/03_bin_decode.t now resets BOTH MAX_COLLECTION_ITEMS and
MAX_COLLECTION_STRUCTURAL to their defaults (they're both initialized from
AVRO_MAX_COLLECTION_ITEMS at load). Verified the whole suite passes with
AVRO_MAX_COLLECTION_ITEMS=5 set.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -255,4 +317,361 @@ EOP
is $dec->{one}[0], 1.0, "kind of dumb test";
}
+## A bytes/string value declares a length prefix; a malicious or truncated
+## input can declare far more bytes than actually exist. On a seekable reader
+## that is rejected before allocating for it.
+{
+ my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" }));
+
+ ## A length prefix declaring 100 MiB, with no payload following it.
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => 100 * 1024 * 1024,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader,
+ );
+ } qr/Cannot read/, "oversized bytes length is rejected before allocating";
+
+ ## A well-formed value above the check threshold whose data is present
+ ## still decodes.
+ my $payload = 'x' x (2 * 1024 * 1024);
+ my $enc2 = '';
+ Avro::BinaryEncoder->encode(
+ schema => $bytes_schema,
+ data => $payload,
+ emit_cb => sub { $enc2 .= ${ $_[0] } },
+ );
+ open my $reader2, '<', \$enc2 or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader2,
+ );
+ is $dec, $payload, "within-limit bytes value still decodes";
+}
+
+## An array/map block declares an element count; a malicious or truncated input
+## can declare far more elements than the remaining bytes could hold. The count
+## is validated against the bytes remaining before iterating, using the minimum
+## on-wire size of the element schema (so 0-byte elements like null are not
+## falsely rejected).
+sub encode_long {
+ my ($value) = @_;
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => $value,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ return $enc;
+}
+
+{
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"long" }));
+ my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized array count is rejected before
iterating";
+}
+
+{
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long"
}));
+ my $enc = encode_long(1_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map count is rejected before
iterating";
+}
+
+{
+ ## An array of nulls: null elements occupy zero bytes, so a large count is
+ ## legitimate and must not be rejected.
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"null" }));
+ my $count = 100_000;
+ my $enc = encode_long($count) . encode_long(0); # one block + end marker
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ is scalar(@$dec), $count, "array of nulls is not falsely rejected";
+}
+
+## Zero-byte elements (null, zero-length fixed, all-null records) consume no
+## input, so the bytes-remaining check cannot bound them. A huge declared block
+## count of such elements is capped instead.
+sub decode_zero_byte_array {
+ my ($items_schema, $enc) = @_;
+ my $schema = Avro::Schema->parse(qq({ "type": "array", "items":
$items_schema }));
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ return Avro::BinaryDecoder->decode(
+ writer_schema => $schema,
+ reader_schema => $schema,
+ reader => $reader,
+ );
+}
+
+{
+ ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is
+ ## rejected by the default limit, before allocating. Pin the limit to the
+ ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS
was
+ ## set in the environment when the module was loaded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS =
+ $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(200_000_000) .
encode_long(0));
+ } qr/more than \d+ zero-byte/, "array of 200M nulls rejected by default
limit";
+}
+
+{
+ ## Within a lowered limit, a legitimate small null array still decodes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ my $dec = decode_zero_byte_array('"null"', encode_long(1000) .
encode_long(0));
+ is scalar(@$dec), 1000, "array of nulls within configured limit still
reads";
+}
+
+{
+ ## INT64_MIN as a block count is the pathological negation case. Negating
it
+ ## yields 2**63, which the cap rejects. INT64_MIN zig-zag encodes as the
+ ## 10-byte varint below, followed by a block byte-size (0).
+ my $int64_min = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01";
+ throws_ok {
+ decode_zero_byte_array('"null"', $int64_min . encode_long(0));
+ } qr/Cannot read a collection|zero-byte|Invalid/, "INT64_MIN array block
count rejected";
+}
+
+{
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(1001) . encode_long(0));
+ } qr/more than 1000 zero-byte/, "array of nulls over configured limit
rejected";
+}
+
+{
+ ## Cumulative across blocks: two blocks of 600 (1200 > 1000).
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
Review Comment:
Addressed globally: rather than localize MAX_COLLECTION_STRUCTURAL per test,
the top of t/03_bin_decode.t now resets BOTH MAX_COLLECTION_ITEMS and
MAX_COLLECTION_STRUCTURAL to their defaults (they're both initialized from
AVRO_MAX_COLLECTION_ITEMS at load). Verified the whole suite passes with
AVRO_MAX_COLLECTION_ITEMS=5 set.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -255,4 +317,361 @@ EOP
is $dec->{one}[0], 1.0, "kind of dumb test";
}
+## A bytes/string value declares a length prefix; a malicious or truncated
+## input can declare far more bytes than actually exist. On a seekable reader
+## that is rejected before allocating for it.
+{
+ my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" }));
+
+ ## A length prefix declaring 100 MiB, with no payload following it.
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => 100 * 1024 * 1024,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader,
+ );
+ } qr/Cannot read/, "oversized bytes length is rejected before allocating";
+
+ ## A well-formed value above the check threshold whose data is present
+ ## still decodes.
+ my $payload = 'x' x (2 * 1024 * 1024);
+ my $enc2 = '';
+ Avro::BinaryEncoder->encode(
+ schema => $bytes_schema,
+ data => $payload,
+ emit_cb => sub { $enc2 .= ${ $_[0] } },
+ );
+ open my $reader2, '<', \$enc2 or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader2,
+ );
+ is $dec, $payload, "within-limit bytes value still decodes";
+}
+
+## An array/map block declares an element count; a malicious or truncated input
+## can declare far more elements than the remaining bytes could hold. The count
+## is validated against the bytes remaining before iterating, using the minimum
+## on-wire size of the element schema (so 0-byte elements like null are not
+## falsely rejected).
+sub encode_long {
+ my ($value) = @_;
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => $value,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ return $enc;
+}
+
+{
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"long" }));
+ my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized array count is rejected before
iterating";
+}
+
+{
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long"
}));
+ my $enc = encode_long(1_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map count is rejected before
iterating";
+}
+
+{
+ ## An array of nulls: null elements occupy zero bytes, so a large count is
+ ## legitimate and must not be rejected.
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"null" }));
+ my $count = 100_000;
+ my $enc = encode_long($count) . encode_long(0); # one block + end marker
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ is scalar(@$dec), $count, "array of nulls is not falsely rejected";
+}
+
+## Zero-byte elements (null, zero-length fixed, all-null records) consume no
+## input, so the bytes-remaining check cannot bound them. A huge declared block
+## count of such elements is capped instead.
+sub decode_zero_byte_array {
+ my ($items_schema, $enc) = @_;
+ my $schema = Avro::Schema->parse(qq({ "type": "array", "items":
$items_schema }));
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ return Avro::BinaryDecoder->decode(
+ writer_schema => $schema,
+ reader_schema => $schema,
+ reader => $reader,
+ );
+}
+
+{
+ ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is
+ ## rejected by the default limit, before allocating. Pin the limit to the
+ ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS
was
+ ## set in the environment when the module was loaded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS =
+ $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(200_000_000) .
encode_long(0));
+ } qr/more than \d+ zero-byte/, "array of 200M nulls rejected by default
limit";
+}
+
+{
+ ## Within a lowered limit, a legitimate small null array still decodes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ my $dec = decode_zero_byte_array('"null"', encode_long(1000) .
encode_long(0));
+ is scalar(@$dec), 1000, "array of nulls within configured limit still
reads";
+}
+
+{
+ ## INT64_MIN as a block count is the pathological negation case. Negating
it
+ ## yields 2**63, which the cap rejects. INT64_MIN zig-zag encodes as the
+ ## 10-byte varint below, followed by a block byte-size (0).
+ my $int64_min = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01";
+ throws_ok {
+ decode_zero_byte_array('"null"', $int64_min . encode_long(0));
+ } qr/Cannot read a collection|zero-byte|Invalid/, "INT64_MIN array block
count rejected";
+}
+
+{
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(1001) . encode_long(0));
+ } qr/more than 1000 zero-byte/, "array of nulls over configured limit
rejected";
+}
+
+{
+ ## Cumulative across blocks: two blocks of 600 (1200 > 1000).
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(600) . encode_long(600) .
encode_long(0));
+ } qr/more than 1000 zero-byte/, "cumulative null blocks over limit
rejected";
+}
+
+{
+ ## A negative block count (abs value + block byte-size) is normalized and
+ ## must still be bounded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
Review Comment:
Addressed globally: rather than localize MAX_COLLECTION_STRUCTURAL per test,
the top of t/03_bin_decode.t now resets BOTH MAX_COLLECTION_ITEMS and
MAX_COLLECTION_STRUCTURAL to their defaults (they're both initialized from
AVRO_MAX_COLLECTION_ITEMS at load). Verified the whole suite passes with
AVRO_MAX_COLLECTION_ITEMS=5 set.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -255,4 +317,361 @@ EOP
is $dec->{one}[0], 1.0, "kind of dumb test";
}
+## A bytes/string value declares a length prefix; a malicious or truncated
+## input can declare far more bytes than actually exist. On a seekable reader
+## that is rejected before allocating for it.
+{
+ my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" }));
+
+ ## A length prefix declaring 100 MiB, with no payload following it.
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => 100 * 1024 * 1024,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader,
+ );
+ } qr/Cannot read/, "oversized bytes length is rejected before allocating";
+
+ ## A well-formed value above the check threshold whose data is present
+ ## still decodes.
+ my $payload = 'x' x (2 * 1024 * 1024);
+ my $enc2 = '';
+ Avro::BinaryEncoder->encode(
+ schema => $bytes_schema,
+ data => $payload,
+ emit_cb => sub { $enc2 .= ${ $_[0] } },
+ );
+ open my $reader2, '<', \$enc2 or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader2,
+ );
+ is $dec, $payload, "within-limit bytes value still decodes";
+}
+
+## An array/map block declares an element count; a malicious or truncated input
+## can declare far more elements than the remaining bytes could hold. The count
+## is validated against the bytes remaining before iterating, using the minimum
+## on-wire size of the element schema (so 0-byte elements like null are not
+## falsely rejected).
+sub encode_long {
+ my ($value) = @_;
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => $value,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ return $enc;
+}
+
+{
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"long" }));
+ my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized array count is rejected before
iterating";
+}
+
+{
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long"
}));
+ my $enc = encode_long(1_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map count is rejected before
iterating";
+}
+
+{
+ ## An array of nulls: null elements occupy zero bytes, so a large count is
+ ## legitimate and must not be rejected.
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"null" }));
+ my $count = 100_000;
+ my $enc = encode_long($count) . encode_long(0); # one block + end marker
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ is scalar(@$dec), $count, "array of nulls is not falsely rejected";
+}
+
+## Zero-byte elements (null, zero-length fixed, all-null records) consume no
+## input, so the bytes-remaining check cannot bound them. A huge declared block
+## count of such elements is capped instead.
+sub decode_zero_byte_array {
+ my ($items_schema, $enc) = @_;
+ my $schema = Avro::Schema->parse(qq({ "type": "array", "items":
$items_schema }));
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ return Avro::BinaryDecoder->decode(
+ writer_schema => $schema,
+ reader_schema => $schema,
+ reader => $reader,
+ );
+}
+
+{
+ ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is
+ ## rejected by the default limit, before allocating. Pin the limit to the
+ ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS
was
+ ## set in the environment when the module was loaded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS =
+ $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(200_000_000) .
encode_long(0));
+ } qr/more than \d+ zero-byte/, "array of 200M nulls rejected by default
limit";
+}
+
+{
+ ## Within a lowered limit, a legitimate small null array still decodes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ my $dec = decode_zero_byte_array('"null"', encode_long(1000) .
encode_long(0));
+ is scalar(@$dec), 1000, "array of nulls within configured limit still
reads";
+}
+
+{
+ ## INT64_MIN as a block count is the pathological negation case. Negating
it
+ ## yields 2**63, which the cap rejects. INT64_MIN zig-zag encodes as the
+ ## 10-byte varint below, followed by a block byte-size (0).
+ my $int64_min = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01";
+ throws_ok {
+ decode_zero_byte_array('"null"', $int64_min . encode_long(0));
+ } qr/Cannot read a collection|zero-byte|Invalid/, "INT64_MIN array block
count rejected";
+}
+
+{
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(1001) . encode_long(0));
+ } qr/more than 1000 zero-byte/, "array of nulls over configured limit
rejected";
+}
+
+{
+ ## Cumulative across blocks: two blocks of 600 (1200 > 1000).
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(600) . encode_long(600) .
encode_long(0));
+ } qr/more than 1000 zero-byte/, "cumulative null blocks over limit
rejected";
+}
+
+{
+ ## A negative block count (abs value + block byte-size) is normalized and
+ ## must still be bounded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(-200_000) .
encode_long(0));
+ } qr/more than 1000 zero-byte/, "negative null block count rejected";
+}
+
+{
+ ## A record whose only field is null encodes to zero bytes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
Review Comment:
Addressed globally: rather than localize MAX_COLLECTION_STRUCTURAL per test,
the top of t/03_bin_decode.t now resets BOTH MAX_COLLECTION_ITEMS and
MAX_COLLECTION_STRUCTURAL to their defaults (they're both initialized from
AVRO_MAX_COLLECTION_ITEMS at load). Verified the whole suite passes with
AVRO_MAX_COLLECTION_ITEMS=5 set.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -255,4 +317,361 @@ EOP
is $dec->{one}[0], 1.0, "kind of dumb test";
}
+## A bytes/string value declares a length prefix; a malicious or truncated
+## input can declare far more bytes than actually exist. On a seekable reader
+## that is rejected before allocating for it.
+{
+ my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" }));
+
+ ## A length prefix declaring 100 MiB, with no payload following it.
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => 100 * 1024 * 1024,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader,
+ );
+ } qr/Cannot read/, "oversized bytes length is rejected before allocating";
+
+ ## A well-formed value above the check threshold whose data is present
+ ## still decodes.
+ my $payload = 'x' x (2 * 1024 * 1024);
+ my $enc2 = '';
+ Avro::BinaryEncoder->encode(
+ schema => $bytes_schema,
+ data => $payload,
+ emit_cb => sub { $enc2 .= ${ $_[0] } },
+ );
+ open my $reader2, '<', \$enc2 or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader2,
+ );
+ is $dec, $payload, "within-limit bytes value still decodes";
+}
+
+## An array/map block declares an element count; a malicious or truncated input
+## can declare far more elements than the remaining bytes could hold. The count
+## is validated against the bytes remaining before iterating, using the minimum
+## on-wire size of the element schema (so 0-byte elements like null are not
+## falsely rejected).
+sub encode_long {
+ my ($value) = @_;
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => $value,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ return $enc;
+}
+
+{
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"long" }));
+ my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized array count is rejected before
iterating";
+}
+
+{
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long"
}));
+ my $enc = encode_long(1_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map count is rejected before
iterating";
+}
+
+{
+ ## An array of nulls: null elements occupy zero bytes, so a large count is
+ ## legitimate and must not be rejected.
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"null" }));
+ my $count = 100_000;
+ my $enc = encode_long($count) . encode_long(0); # one block + end marker
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ is scalar(@$dec), $count, "array of nulls is not falsely rejected";
+}
+
+## Zero-byte elements (null, zero-length fixed, all-null records) consume no
+## input, so the bytes-remaining check cannot bound them. A huge declared block
+## count of such elements is capped instead.
+sub decode_zero_byte_array {
+ my ($items_schema, $enc) = @_;
+ my $schema = Avro::Schema->parse(qq({ "type": "array", "items":
$items_schema }));
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ return Avro::BinaryDecoder->decode(
+ writer_schema => $schema,
+ reader_schema => $schema,
+ reader => $reader,
+ );
+}
+
+{
+ ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is
+ ## rejected by the default limit, before allocating. Pin the limit to the
+ ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS
was
+ ## set in the environment when the module was loaded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS =
+ $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(200_000_000) .
encode_long(0));
+ } qr/more than \d+ zero-byte/, "array of 200M nulls rejected by default
limit";
+}
+
+{
+ ## Within a lowered limit, a legitimate small null array still decodes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ my $dec = decode_zero_byte_array('"null"', encode_long(1000) .
encode_long(0));
+ is scalar(@$dec), 1000, "array of nulls within configured limit still
reads";
+}
+
+{
+ ## INT64_MIN as a block count is the pathological negation case. Negating
it
+ ## yields 2**63, which the cap rejects. INT64_MIN zig-zag encodes as the
+ ## 10-byte varint below, followed by a block byte-size (0).
+ my $int64_min = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01";
+ throws_ok {
+ decode_zero_byte_array('"null"', $int64_min . encode_long(0));
+ } qr/Cannot read a collection|zero-byte|Invalid/, "INT64_MIN array block
count rejected";
+}
+
+{
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(1001) . encode_long(0));
+ } qr/more than 1000 zero-byte/, "array of nulls over configured limit
rejected";
+}
+
+{
+ ## Cumulative across blocks: two blocks of 600 (1200 > 1000).
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(600) . encode_long(600) .
encode_long(0));
+ } qr/more than 1000 zero-byte/, "cumulative null blocks over limit
rejected";
+}
+
+{
+ ## A negative block count (abs value + block byte-size) is normalized and
+ ## must still be bounded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(-200_000) .
encode_long(0));
+ } qr/more than 1000 zero-byte/, "negative null block count rejected";
+}
+
+{
+ ## A record whose only field is null encodes to zero bytes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('{ "type": "record", "name": "R", "fields":
[{"name":"n","type":"null"}] }',
+ encode_long(2000) . encode_long(0));
+ } qr/more than 1000 zero-byte/, "array of all-null records over limit
rejected";
+}
+
+{
+ ## A huge map<null> is bounded by the bytes-remaining check (each entry
has a
+ ## >= 1 byte key), not the zero-byte cap.
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "null"
}));
+ my $enc = encode_long(200_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map<null> rejected by available-bytes
check";
+}
+
+{
+ ## Skipping a huge array<null> writer field absent from the reader schema
is
+ ## bounded (a CPU exhaustion otherwise), via skip_block.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
Review Comment:
Addressed globally: rather than localize MAX_COLLECTION_STRUCTURAL per test,
the top of t/03_bin_decode.t now resets BOTH MAX_COLLECTION_ITEMS and
MAX_COLLECTION_STRUCTURAL to their defaults (they're both initialized from
AVRO_MAX_COLLECTION_ITEMS at load). Verified the whole suite passes with
AVRO_MAX_COLLECTION_ITEMS=5 set.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -255,4 +317,361 @@ EOP
is $dec->{one}[0], 1.0, "kind of dumb test";
}
+## A bytes/string value declares a length prefix; a malicious or truncated
+## input can declare far more bytes than actually exist. On a seekable reader
+## that is rejected before allocating for it.
+{
+ my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" }));
+
+ ## A length prefix declaring 100 MiB, with no payload following it.
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => 100 * 1024 * 1024,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader,
+ );
+ } qr/Cannot read/, "oversized bytes length is rejected before allocating";
+
+ ## A well-formed value above the check threshold whose data is present
+ ## still decodes.
+ my $payload = 'x' x (2 * 1024 * 1024);
+ my $enc2 = '';
+ Avro::BinaryEncoder->encode(
+ schema => $bytes_schema,
+ data => $payload,
+ emit_cb => sub { $enc2 .= ${ $_[0] } },
+ );
+ open my $reader2, '<', \$enc2 or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader2,
+ );
+ is $dec, $payload, "within-limit bytes value still decodes";
+}
+
+## An array/map block declares an element count; a malicious or truncated input
+## can declare far more elements than the remaining bytes could hold. The count
+## is validated against the bytes remaining before iterating, using the minimum
+## on-wire size of the element schema (so 0-byte elements like null are not
+## falsely rejected).
+sub encode_long {
+ my ($value) = @_;
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => $value,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ return $enc;
+}
+
+{
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"long" }));
+ my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized array count is rejected before
iterating";
+}
+
+{
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long"
}));
+ my $enc = encode_long(1_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map count is rejected before
iterating";
+}
+
+{
+ ## An array of nulls: null elements occupy zero bytes, so a large count is
+ ## legitimate and must not be rejected.
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"null" }));
+ my $count = 100_000;
+ my $enc = encode_long($count) . encode_long(0); # one block + end marker
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ is scalar(@$dec), $count, "array of nulls is not falsely rejected";
+}
+
+## Zero-byte elements (null, zero-length fixed, all-null records) consume no
+## input, so the bytes-remaining check cannot bound them. A huge declared block
+## count of such elements is capped instead.
+sub decode_zero_byte_array {
+ my ($items_schema, $enc) = @_;
+ my $schema = Avro::Schema->parse(qq({ "type": "array", "items":
$items_schema }));
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ return Avro::BinaryDecoder->decode(
+ writer_schema => $schema,
+ reader_schema => $schema,
+ reader => $reader,
+ );
+}
+
+{
+ ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is
+ ## rejected by the default limit, before allocating. Pin the limit to the
+ ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS
was
+ ## set in the environment when the module was loaded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS =
+ $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(200_000_000) .
encode_long(0));
+ } qr/more than \d+ zero-byte/, "array of 200M nulls rejected by default
limit";
+}
+
+{
+ ## Within a lowered limit, a legitimate small null array still decodes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ my $dec = decode_zero_byte_array('"null"', encode_long(1000) .
encode_long(0));
+ is scalar(@$dec), 1000, "array of nulls within configured limit still
reads";
+}
+
+{
+ ## INT64_MIN as a block count is the pathological negation case. Negating
it
+ ## yields 2**63, which the cap rejects. INT64_MIN zig-zag encodes as the
+ ## 10-byte varint below, followed by a block byte-size (0).
+ my $int64_min = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01";
+ throws_ok {
+ decode_zero_byte_array('"null"', $int64_min . encode_long(0));
+ } qr/Cannot read a collection|zero-byte|Invalid/, "INT64_MIN array block
count rejected";
+}
+
+{
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(1001) . encode_long(0));
+ } qr/more than 1000 zero-byte/, "array of nulls over configured limit
rejected";
+}
+
+{
+ ## Cumulative across blocks: two blocks of 600 (1200 > 1000).
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(600) . encode_long(600) .
encode_long(0));
+ } qr/more than 1000 zero-byte/, "cumulative null blocks over limit
rejected";
+}
+
+{
+ ## A negative block count (abs value + block byte-size) is normalized and
+ ## must still be bounded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(-200_000) .
encode_long(0));
+ } qr/more than 1000 zero-byte/, "negative null block count rejected";
+}
+
+{
+ ## A record whose only field is null encodes to zero bytes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('{ "type": "record", "name": "R", "fields":
[{"name":"n","type":"null"}] }',
+ encode_long(2000) . encode_long(0));
+ } qr/more than 1000 zero-byte/, "array of all-null records over limit
rejected";
+}
+
+{
+ ## A huge map<null> is bounded by the bytes-remaining check (each entry
has a
+ ## >= 1 byte key), not the zero-byte cap.
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "null"
}));
+ my $enc = encode_long(200_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map<null> rejected by available-bytes
check";
+}
+
+{
+ ## Skipping a huge array<null> writer field absent from the reader schema
is
+ ## bounded (a CPU exhaustion otherwise), via skip_block.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ my $w_schema = Avro::Schema->parse(<<'EOJ');
+ { "type": "record", "name": "test",
+ "fields" : [
+ {"name": "arr", "type": {"type": "array", "items": "null"}},
+ {"name": "a", "type": "long"} ]}
+EOJ
+ my $r_schema = Avro::Schema->parse(<<'EOJ');
+ { "type": "record", "name": "test",
+ "fields" : [ {"name": "a", "type": "long"} ]}
+EOJ
+ # Hand-crafted: array block of 2000 nulls + end marker, then a = 42.
+ my $enc = encode_long(2000) . encode_long(0) . encode_long(42);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $w_schema,
+ reader_schema => $r_schema,
+ reader => $reader,
+ );
+ } qr/more than 1000/, "skipping oversized array<null> field is bounded";
+}
+
+{
+ ## The zero-byte cap raises the dedicated exception class.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
Review Comment:
Addressed globally: rather than localize MAX_COLLECTION_STRUCTURAL per test,
the top of t/03_bin_decode.t now resets BOTH MAX_COLLECTION_ITEMS and
MAX_COLLECTION_STRUCTURAL to their defaults (they're both initialized from
AVRO_MAX_COLLECTION_ITEMS at load). Verified the whole suite passes with
AVRO_MAX_COLLECTION_ITEMS=5 set.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
