Copilot commented on code in PR #3864:
URL: https://github.com/apache/avro/pull/3864#discussion_r3567640154
##########
lang/perl/lib/Avro/BinaryDecoder.pm:
##########
@@ -215,27 +387,78 @@ sub decode_enum {
}
sub skip_block {
- my $class = shift;
- my ($reader, $block_content) = @_;
- my $block_count = decode_long($class, undef, undef, $reader);
+ # Called as a plain function by skip_array/skip_map as
+ # skip_block($reader, $min_bytes, $coderef); it is not a method, so there
is
+ # no leading class argument. decode_long ignores its (shifted) first
argument
+ # and reads from the last, so pass __PACKAGE__ for it and $reader last.
+ my ($reader, $min_bytes, $block_content) = @_;
+ # Zero-byte elements loop with no per-item input, so bound them tightly;
+ # other elements consume bytes (bounded by EOF) so only the structural cap
+ # applies. Every collection is capped by the structural limit, so for
+ # zero-byte elements use whichever of the two limits is tighter.
+ my $limit = $MAX_COLLECTION_STRUCTURAL;
+ if ($min_bytes <= 0 && $MAX_COLLECTION_ITEMS < $limit) {
+ $limit = $MAX_COLLECTION_ITEMS;
+ }
+ my $block_count = decode_long(__PACKAGE__, undef, undef, $reader);
+ my $skipped = 0;
while ($block_count) {
if ($block_count < 0) {
- $reader->seek($block_count, 0);
- next;
+ # A byte-sized (negative-count) block still declares an element
+ # count; bound it cumulatively too, so a huge declared count with a
+ # small or zero block size cannot bypass the cap (e.g. zero-byte
+ # elements whose block size is 0).
+ my $count = -$block_count;
+ if ($skipped + $count > $limit) {
+ throw Avro::BinaryDecoder::Error::CollectionSize(
+ "Cannot skip a collection of more than $limit "
+ . "elements (declared @{[ $skipped + $count ]})");
+ }
+ $skipped += $count;
+ # A negative count is followed by a long block size in bytes, which
+ # lets the whole block be skipped without decoding each item. Skip
+ # forward by that many bytes relative to the current position
+ # (SEEK_CUR); whence 0 (SEEK_SET) would seek to an absolute (here
+ # nonsensical) offset. A failed seek is treated as fatal.
+ my $block_size = decode_long(__PACKAGE__, undef, undef, $reader);
+ if ($block_size < 0) {
+ # A negative block size would seek the reader backwards,
+ # risking an infinite loop or mis-decoding of a corrupt input.
+ throw Avro::Schema::Error::Parse(
+ "Invalid negative block size: $block_size");
+ }
+ # Reject a block size that exceeds the bytes actually remaining
+ # before skipping, so a truncated input fails instead of seeking
+ # past EOF.
+ _ensure_available($reader, $block_size, 1);
+ unless ($reader->seek($block_size, 1)) {
+ throw Avro::Schema::Error::Parse(
+ "Failed to skip block of $block_size bytes");
+ }
}
else {
+ # Bound the cumulative element count: skipping a huge block of
+ # zero-byte elements would otherwise loop unboundedly (a CPU
+ # exhaustion) even though it allocates nothing.
+ if ($skipped + $block_count > $limit) {
+ throw Avro::BinaryDecoder::Error::CollectionSize(
+ "Cannot skip a collection of more than $limit "
+ . "elements (declared @{[ $skipped + $block_count ]})");
+ }
+ $skipped += $block_count;
for (1..$block_count) {
$block_content->();
}
Review Comment:
`skip_block` uses `for (1..$block_count)`, which materializes a list of all
integers in the range before iterating. With large declared block counts
(especially on non-seekable readers where only the structural cap applies),
this can cause immediate memory/CPU exhaustion even if decoding would fail
later. Use an index-based loop that doesn't allocate the range list.
##########
lang/perl/lib/Avro/BinaryDecoder.pm:
##########
@@ -296,13 +526,27 @@ sub decode_map {
my $block_count = decode_long($class, @_);
my $writer_values = $writer_schema->values;
my $reader_values = $reader_schema->values;
+ # A map key is a non-empty string (decode_map rejects empty keys below), so
+ # its minimum on-wire size is 2 bytes: a 1-byte length prefix plus at least
+ # 1 byte of key data, in addition to the value.
+ my $min_bytes = 2 + _min_bytes_per_element($writer_values);
+ # Track the number of decoded entries separately: scalar(keys %hash)
+ # undercounts when duplicate keys overwrite earlier entries, which would
let
+ # a multi-block map exceed the cumulative caps without being rejected.
+ my $decoded = 0;
while ($block_count) {
my $block_size;
if ($block_count < 0) {
$block_count = -$block_count;
$block_size = decode_long($class, @_);
+ if ($block_size < 0) {
+ throw Avro::Schema::Error::Parse(
+ "Invalid negative map block size: $block_size");
+ }
## XXX we can skip with $reader_schema?
}
+ _ensure_collection_available($reader, $decoded, $block_count,
$min_bytes);
+ $decoded += $block_count;
for (1..$block_count) {
my $key = decode_string($class, @_);
unless (defined $key && length $key) {
Review Comment:
`decode_map` uses `for (1..$block_count)`, which materializes a list of the
whole range before iterating. A large declared block count can therefore cause
memory/CPU exhaustion even before any key/value decoding occurs (and even if
decoding would fail later). Prefer an index-based loop that doesn't allocate
the range list.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -255,4 +317,361 @@ EOP
is $dec->{one}[0], 1.0, "kind of dumb test";
}
+## A bytes/string value declares a length prefix; a malicious or truncated
+## input can declare far more bytes than actually exist. On a seekable reader
+## that is rejected before allocating for it.
+{
+ my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" }));
+
+ ## A length prefix declaring 100 MiB, with no payload following it.
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => 100 * 1024 * 1024,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader,
+ );
+ } qr/Cannot read/, "oversized bytes length is rejected before allocating";
+
+ ## A well-formed value above the check threshold whose data is present
+ ## still decodes.
+ my $payload = 'x' x (2 * 1024 * 1024);
+ my $enc2 = '';
+ Avro::BinaryEncoder->encode(
+ schema => $bytes_schema,
+ data => $payload,
+ emit_cb => sub { $enc2 .= ${ $_[0] } },
+ );
+ open my $reader2, '<', \$enc2 or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader2,
+ );
+ is $dec, $payload, "within-limit bytes value still decodes";
+}
+
+## An array/map block declares an element count; a malicious or truncated input
+## can declare far more elements than the remaining bytes could hold. The count
+## is validated against the bytes remaining before iterating, using the minimum
+## on-wire size of the element schema (so 0-byte elements like null are not
+## falsely rejected).
+sub encode_long {
+ my ($value) = @_;
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => $value,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ return $enc;
+}
+
+{
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"long" }));
+ my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized array count is rejected before
iterating";
+}
+
+{
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long"
}));
+ my $enc = encode_long(1_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map count is rejected before
iterating";
+}
+
+{
+ ## An array of nulls: null elements occupy zero bytes, so a large count is
+ ## legitimate and must not be rejected.
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"null" }));
+ my $count = 100_000;
+ my $enc = encode_long($count) . encode_long(0); # one block + end marker
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ is scalar(@$dec), $count, "array of nulls is not falsely rejected";
+}
+
+## Zero-byte elements (null, zero-length fixed, all-null records) consume no
+## input, so the bytes-remaining check cannot bound them. A huge declared block
+## count of such elements is capped instead.
+sub decode_zero_byte_array {
+ my ($items_schema, $enc) = @_;
+ my $schema = Avro::Schema->parse(qq({ "type": "array", "items":
$items_schema }));
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ return Avro::BinaryDecoder->decode(
+ writer_schema => $schema,
+ reader_schema => $schema,
+ reader => $reader,
+ );
+}
+
+{
+ ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is
+ ## rejected by the default limit, before allocating. Pin the limit to the
+ ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS
was
+ ## set in the environment when the module was loaded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS =
+ $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS;
Review Comment:
This block aims to be deterministic when `AVRO_MAX_COLLECTION_ITEMS` is set
in the environment, but that env var also initializes
`MAX_COLLECTION_STRUCTURAL`. If the env var is smaller than the defaults, the
structural cap can fire first and change the exception/message, making the test
non-deterministic. Localize `MAX_COLLECTION_STRUCTURAL` to the default here as
well.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -255,4 +317,361 @@ EOP
is $dec->{one}[0], 1.0, "kind of dumb test";
}
+## A bytes/string value declares a length prefix; a malicious or truncated
+## input can declare far more bytes than actually exist. On a seekable reader
+## that is rejected before allocating for it.
+{
+ my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" }));
+
+ ## A length prefix declaring 100 MiB, with no payload following it.
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => 100 * 1024 * 1024,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader,
+ );
+ } qr/Cannot read/, "oversized bytes length is rejected before allocating";
+
+ ## A well-formed value above the check threshold whose data is present
+ ## still decodes.
+ my $payload = 'x' x (2 * 1024 * 1024);
+ my $enc2 = '';
+ Avro::BinaryEncoder->encode(
+ schema => $bytes_schema,
+ data => $payload,
+ emit_cb => sub { $enc2 .= ${ $_[0] } },
+ );
+ open my $reader2, '<', \$enc2 or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader2,
+ );
+ is $dec, $payload, "within-limit bytes value still decodes";
+}
+
+## An array/map block declares an element count; a malicious or truncated input
+## can declare far more elements than the remaining bytes could hold. The count
+## is validated against the bytes remaining before iterating, using the minimum
+## on-wire size of the element schema (so 0-byte elements like null are not
+## falsely rejected).
+sub encode_long {
+ my ($value) = @_;
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => $value,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ return $enc;
+}
+
+{
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"long" }));
+ my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized array count is rejected before
iterating";
+}
+
+{
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long"
}));
+ my $enc = encode_long(1_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map count is rejected before
iterating";
+}
+
+{
+ ## An array of nulls: null elements occupy zero bytes, so a large count is
+ ## legitimate and must not be rejected.
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"null" }));
+ my $count = 100_000;
+ my $enc = encode_long($count) . encode_long(0); # one block + end marker
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ is scalar(@$dec), $count, "array of nulls is not falsely rejected";
+}
+
+## Zero-byte elements (null, zero-length fixed, all-null records) consume no
+## input, so the bytes-remaining check cannot bound them. A huge declared block
+## count of such elements is capped instead.
+sub decode_zero_byte_array {
+ my ($items_schema, $enc) = @_;
+ my $schema = Avro::Schema->parse(qq({ "type": "array", "items":
$items_schema }));
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ return Avro::BinaryDecoder->decode(
+ writer_schema => $schema,
+ reader_schema => $schema,
+ reader => $reader,
+ );
+}
+
+{
+ ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is
+ ## rejected by the default limit, before allocating. Pin the limit to the
+ ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS
was
+ ## set in the environment when the module was loaded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS =
+ $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(200_000_000) .
encode_long(0));
+ } qr/more than \d+ zero-byte/, "array of 200M nulls rejected by default
limit";
+}
+
+{
+ ## Within a lowered limit, a legitimate small null array still decodes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
Review Comment:
This test sets `MAX_COLLECTION_ITEMS` but not `MAX_COLLECTION_STRUCTURAL`.
Since `AVRO_MAX_COLLECTION_ITEMS` initializes both caps at module load time, a
smaller structural cap can trigger first and change the failure mode, making
the test dependent on the environment. Localize `MAX_COLLECTION_STRUCTURAL` to
the default alongside `MAX_COLLECTION_ITEMS`.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -255,4 +317,361 @@ EOP
is $dec->{one}[0], 1.0, "kind of dumb test";
}
+## A bytes/string value declares a length prefix; a malicious or truncated
+## input can declare far more bytes than actually exist. On a seekable reader
+## that is rejected before allocating for it.
+{
+ my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" }));
+
+ ## A length prefix declaring 100 MiB, with no payload following it.
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => 100 * 1024 * 1024,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader,
+ );
+ } qr/Cannot read/, "oversized bytes length is rejected before allocating";
+
+ ## A well-formed value above the check threshold whose data is present
+ ## still decodes.
+ my $payload = 'x' x (2 * 1024 * 1024);
+ my $enc2 = '';
+ Avro::BinaryEncoder->encode(
+ schema => $bytes_schema,
+ data => $payload,
+ emit_cb => sub { $enc2 .= ${ $_[0] } },
+ );
+ open my $reader2, '<', \$enc2 or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader2,
+ );
+ is $dec, $payload, "within-limit bytes value still decodes";
+}
+
+## An array/map block declares an element count; a malicious or truncated input
+## can declare far more elements than the remaining bytes could hold. The count
+## is validated against the bytes remaining before iterating, using the minimum
+## on-wire size of the element schema (so 0-byte elements like null are not
+## falsely rejected).
+sub encode_long {
+ my ($value) = @_;
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => $value,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ return $enc;
+}
+
+{
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"long" }));
+ my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized array count is rejected before
iterating";
+}
+
+{
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long"
}));
+ my $enc = encode_long(1_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map count is rejected before
iterating";
+}
+
+{
+ ## An array of nulls: null elements occupy zero bytes, so a large count is
+ ## legitimate and must not be rejected.
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"null" }));
+ my $count = 100_000;
+ my $enc = encode_long($count) . encode_long(0); # one block + end marker
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ is scalar(@$dec), $count, "array of nulls is not falsely rejected";
+}
+
+## Zero-byte elements (null, zero-length fixed, all-null records) consume no
+## input, so the bytes-remaining check cannot bound them. A huge declared block
+## count of such elements is capped instead.
+sub decode_zero_byte_array {
+ my ($items_schema, $enc) = @_;
+ my $schema = Avro::Schema->parse(qq({ "type": "array", "items":
$items_schema }));
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ return Avro::BinaryDecoder->decode(
+ writer_schema => $schema,
+ reader_schema => $schema,
+ reader => $reader,
+ );
+}
+
+{
+ ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is
+ ## rejected by the default limit, before allocating. Pin the limit to the
+ ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS
was
+ ## set in the environment when the module was loaded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS =
+ $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(200_000_000) .
encode_long(0));
+ } qr/more than \d+ zero-byte/, "array of 200M nulls rejected by default
limit";
+}
+
+{
+ ## Within a lowered limit, a legitimate small null array still decodes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ my $dec = decode_zero_byte_array('"null"', encode_long(1000) .
encode_long(0));
+ is scalar(@$dec), 1000, "array of nulls within configured limit still
reads";
+}
+
+{
+ ## INT64_MIN as a block count is the pathological negation case. Negating
it
+ ## yields 2**63, which the cap rejects. INT64_MIN zig-zag encodes as the
+ ## 10-byte varint below, followed by a block byte-size (0).
+ my $int64_min = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01";
+ throws_ok {
+ decode_zero_byte_array('"null"', $int64_min . encode_long(0));
+ } qr/Cannot read a collection|zero-byte|Invalid/, "INT64_MIN array block
count rejected";
+}
+
+{
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(1001) . encode_long(0));
+ } qr/more than 1000 zero-byte/, "array of nulls over configured limit
rejected";
+}
+
+{
+ ## Cumulative across blocks: two blocks of 600 (1200 > 1000).
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
Review Comment:
This test assumes the zero-byte item cap is the active bound, but
`AVRO_MAX_COLLECTION_ITEMS` also sets `MAX_COLLECTION_STRUCTURAL` at load time.
If the structural cap is smaller than 1000, it can trigger first and change the
observed error/message. Localize `MAX_COLLECTION_STRUCTURAL` to the default
here.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -255,4 +317,361 @@ EOP
is $dec->{one}[0], 1.0, "kind of dumb test";
}
+## A bytes/string value declares a length prefix; a malicious or truncated
+## input can declare far more bytes than actually exist. On a seekable reader
+## that is rejected before allocating for it.
+{
+ my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" }));
+
+ ## A length prefix declaring 100 MiB, with no payload following it.
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => 100 * 1024 * 1024,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader,
+ );
+ } qr/Cannot read/, "oversized bytes length is rejected before allocating";
+
+ ## A well-formed value above the check threshold whose data is present
+ ## still decodes.
+ my $payload = 'x' x (2 * 1024 * 1024);
+ my $enc2 = '';
+ Avro::BinaryEncoder->encode(
+ schema => $bytes_schema,
+ data => $payload,
+ emit_cb => sub { $enc2 .= ${ $_[0] } },
+ );
+ open my $reader2, '<', \$enc2 or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader2,
+ );
+ is $dec, $payload, "within-limit bytes value still decodes";
+}
+
+## An array/map block declares an element count; a malicious or truncated input
+## can declare far more elements than the remaining bytes could hold. The count
+## is validated against the bytes remaining before iterating, using the minimum
+## on-wire size of the element schema (so 0-byte elements like null are not
+## falsely rejected).
+sub encode_long {
+ my ($value) = @_;
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => $value,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ return $enc;
+}
+
+{
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"long" }));
+ my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized array count is rejected before
iterating";
+}
+
+{
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long"
}));
+ my $enc = encode_long(1_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map count is rejected before
iterating";
+}
+
+{
+ ## An array of nulls: null elements occupy zero bytes, so a large count is
+ ## legitimate and must not be rejected.
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"null" }));
+ my $count = 100_000;
+ my $enc = encode_long($count) . encode_long(0); # one block + end marker
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ is scalar(@$dec), $count, "array of nulls is not falsely rejected";
+}
+
+## Zero-byte elements (null, zero-length fixed, all-null records) consume no
+## input, so the bytes-remaining check cannot bound them. A huge declared block
+## count of such elements is capped instead.
+sub decode_zero_byte_array {
+ my ($items_schema, $enc) = @_;
+ my $schema = Avro::Schema->parse(qq({ "type": "array", "items":
$items_schema }));
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ return Avro::BinaryDecoder->decode(
+ writer_schema => $schema,
+ reader_schema => $schema,
+ reader => $reader,
+ );
+}
+
+{
+ ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is
+ ## rejected by the default limit, before allocating. Pin the limit to the
+ ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS
was
+ ## set in the environment when the module was loaded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS =
+ $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(200_000_000) .
encode_long(0));
+ } qr/more than \d+ zero-byte/, "array of 200M nulls rejected by default
limit";
+}
+
+{
+ ## Within a lowered limit, a legitimate small null array still decodes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ my $dec = decode_zero_byte_array('"null"', encode_long(1000) .
encode_long(0));
+ is scalar(@$dec), 1000, "array of nulls within configured limit still
reads";
+}
+
+{
+ ## INT64_MIN as a block count is the pathological negation case. Negating
it
+ ## yields 2**63, which the cap rejects. INT64_MIN zig-zag encodes as the
+ ## 10-byte varint below, followed by a block byte-size (0).
+ my $int64_min = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01";
+ throws_ok {
+ decode_zero_byte_array('"null"', $int64_min . encode_long(0));
+ } qr/Cannot read a collection|zero-byte|Invalid/, "INT64_MIN array block
count rejected";
+}
+
+{
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
Review Comment:
This block localizes `MAX_COLLECTION_ITEMS` but not
`MAX_COLLECTION_STRUCTURAL`. Because the env var can set a smaller structural
cap at module load time, the structural limit may fire instead of the intended
zero-byte item cap, making the test non-deterministic. Localize
`MAX_COLLECTION_STRUCTURAL` to the default in this block too.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -255,4 +317,361 @@ EOP
is $dec->{one}[0], 1.0, "kind of dumb test";
}
+## A bytes/string value declares a length prefix; a malicious or truncated
+## input can declare far more bytes than actually exist. On a seekable reader
+## that is rejected before allocating for it.
+{
+ my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" }));
+
+ ## A length prefix declaring 100 MiB, with no payload following it.
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => 100 * 1024 * 1024,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader,
+ );
+ } qr/Cannot read/, "oversized bytes length is rejected before allocating";
+
+ ## A well-formed value above the check threshold whose data is present
+ ## still decodes.
+ my $payload = 'x' x (2 * 1024 * 1024);
+ my $enc2 = '';
+ Avro::BinaryEncoder->encode(
+ schema => $bytes_schema,
+ data => $payload,
+ emit_cb => sub { $enc2 .= ${ $_[0] } },
+ );
+ open my $reader2, '<', \$enc2 or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader2,
+ );
+ is $dec, $payload, "within-limit bytes value still decodes";
+}
+
+## An array/map block declares an element count; a malicious or truncated input
+## can declare far more elements than the remaining bytes could hold. The count
+## is validated against the bytes remaining before iterating, using the minimum
+## on-wire size of the element schema (so 0-byte elements like null are not
+## falsely rejected).
+sub encode_long {
+ my ($value) = @_;
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => $value,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ return $enc;
+}
+
+{
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"long" }));
+ my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized array count is rejected before
iterating";
+}
+
+{
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long"
}));
+ my $enc = encode_long(1_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map count is rejected before
iterating";
+}
+
+{
+ ## An array of nulls: null elements occupy zero bytes, so a large count is
+ ## legitimate and must not be rejected.
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"null" }));
+ my $count = 100_000;
+ my $enc = encode_long($count) . encode_long(0); # one block + end marker
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ is scalar(@$dec), $count, "array of nulls is not falsely rejected";
+}
+
+## Zero-byte elements (null, zero-length fixed, all-null records) consume no
+## input, so the bytes-remaining check cannot bound them. A huge declared block
+## count of such elements is capped instead.
+sub decode_zero_byte_array {
+ my ($items_schema, $enc) = @_;
+ my $schema = Avro::Schema->parse(qq({ "type": "array", "items":
$items_schema }));
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ return Avro::BinaryDecoder->decode(
+ writer_schema => $schema,
+ reader_schema => $schema,
+ reader => $reader,
+ );
+}
+
+{
+ ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is
+ ## rejected by the default limit, before allocating. Pin the limit to the
+ ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS
was
+ ## set in the environment when the module was loaded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS =
+ $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(200_000_000) .
encode_long(0));
+ } qr/more than \d+ zero-byte/, "array of 200M nulls rejected by default
limit";
+}
+
+{
+ ## Within a lowered limit, a legitimate small null array still decodes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ my $dec = decode_zero_byte_array('"null"', encode_long(1000) .
encode_long(0));
+ is scalar(@$dec), 1000, "array of nulls within configured limit still
reads";
+}
+
+{
+ ## INT64_MIN as a block count is the pathological negation case. Negating
it
+ ## yields 2**63, which the cap rejects. INT64_MIN zig-zag encodes as the
+ ## 10-byte varint below, followed by a block byte-size (0).
+ my $int64_min = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01";
+ throws_ok {
+ decode_zero_byte_array('"null"', $int64_min . encode_long(0));
+ } qr/Cannot read a collection|zero-byte|Invalid/, "INT64_MIN array block
count rejected";
+}
+
+{
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(1001) . encode_long(0));
+ } qr/more than 1000 zero-byte/, "array of nulls over configured limit
rejected";
+}
+
+{
+ ## Cumulative across blocks: two blocks of 600 (1200 > 1000).
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(600) . encode_long(600) .
encode_long(0));
+ } qr/more than 1000 zero-byte/, "cumulative null blocks over limit
rejected";
+}
+
+{
+ ## A negative block count (abs value + block byte-size) is normalized and
+ ## must still be bounded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(-200_000) .
encode_long(0));
+ } qr/more than 1000 zero-byte/, "negative null block count rejected";
+}
+
+{
+ ## A record whose only field is null encodes to zero bytes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
Review Comment:
This test depends on the zero-byte item cap behavior but doesn't localize
`MAX_COLLECTION_STRUCTURAL`. Since the env var initializes both caps at module
load time, a smaller structural cap can cause this test to fail for the wrong
reason. Localize `MAX_COLLECTION_STRUCTURAL` to the default in this block too.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -255,4 +317,361 @@ EOP
is $dec->{one}[0], 1.0, "kind of dumb test";
}
+## A bytes/string value declares a length prefix; a malicious or truncated
+## input can declare far more bytes than actually exist. On a seekable reader
+## that is rejected before allocating for it.
+{
+ my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" }));
+
+ ## A length prefix declaring 100 MiB, with no payload following it.
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => 100 * 1024 * 1024,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader,
+ );
+ } qr/Cannot read/, "oversized bytes length is rejected before allocating";
+
+ ## A well-formed value above the check threshold whose data is present
+ ## still decodes.
+ my $payload = 'x' x (2 * 1024 * 1024);
+ my $enc2 = '';
+ Avro::BinaryEncoder->encode(
+ schema => $bytes_schema,
+ data => $payload,
+ emit_cb => sub { $enc2 .= ${ $_[0] } },
+ );
+ open my $reader2, '<', \$enc2 or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader2,
+ );
+ is $dec, $payload, "within-limit bytes value still decodes";
+}
+
+## An array/map block declares an element count; a malicious or truncated input
+## can declare far more elements than the remaining bytes could hold. The count
+## is validated against the bytes remaining before iterating, using the minimum
+## on-wire size of the element schema (so 0-byte elements like null are not
+## falsely rejected).
+sub encode_long {
+ my ($value) = @_;
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => $value,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ return $enc;
+}
+
+{
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"long" }));
+ my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized array count is rejected before
iterating";
+}
+
+{
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long"
}));
+ my $enc = encode_long(1_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map count is rejected before
iterating";
+}
+
+{
+ ## An array of nulls: null elements occupy zero bytes, so a large count is
+ ## legitimate and must not be rejected.
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"null" }));
+ my $count = 100_000;
+ my $enc = encode_long($count) . encode_long(0); # one block + end marker
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ is scalar(@$dec), $count, "array of nulls is not falsely rejected";
+}
+
+## Zero-byte elements (null, zero-length fixed, all-null records) consume no
+## input, so the bytes-remaining check cannot bound them. A huge declared block
+## count of such elements is capped instead.
+sub decode_zero_byte_array {
+ my ($items_schema, $enc) = @_;
+ my $schema = Avro::Schema->parse(qq({ "type": "array", "items":
$items_schema }));
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ return Avro::BinaryDecoder->decode(
+ writer_schema => $schema,
+ reader_schema => $schema,
+ reader => $reader,
+ );
+}
+
+{
+ ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is
+ ## rejected by the default limit, before allocating. Pin the limit to the
+ ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS
was
+ ## set in the environment when the module was loaded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS =
+ $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(200_000_000) .
encode_long(0));
+ } qr/more than \d+ zero-byte/, "array of 200M nulls rejected by default
limit";
+}
+
+{
+ ## Within a lowered limit, a legitimate small null array still decodes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ my $dec = decode_zero_byte_array('"null"', encode_long(1000) .
encode_long(0));
+ is scalar(@$dec), 1000, "array of nulls within configured limit still
reads";
+}
+
+{
+ ## INT64_MIN as a block count is the pathological negation case. Negating
it
+ ## yields 2**63, which the cap rejects. INT64_MIN zig-zag encodes as the
+ ## 10-byte varint below, followed by a block byte-size (0).
+ my $int64_min = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01";
+ throws_ok {
+ decode_zero_byte_array('"null"', $int64_min . encode_long(0));
+ } qr/Cannot read a collection|zero-byte|Invalid/, "INT64_MIN array block
count rejected";
+}
+
+{
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(1001) . encode_long(0));
+ } qr/more than 1000 zero-byte/, "array of nulls over configured limit
rejected";
+}
+
+{
+ ## Cumulative across blocks: two blocks of 600 (1200 > 1000).
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(600) . encode_long(600) .
encode_long(0));
+ } qr/more than 1000 zero-byte/, "cumulative null blocks over limit
rejected";
+}
+
+{
+ ## A negative block count (abs value + block byte-size) is normalized and
+ ## must still be bounded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
Review Comment:
This block localizes only `MAX_COLLECTION_ITEMS`. If
`MAX_COLLECTION_STRUCTURAL` was initialized to a smaller value via
`AVRO_MAX_COLLECTION_ITEMS`, the structural cap may fire first and make this
test depend on the environment. Localize `MAX_COLLECTION_STRUCTURAL` to the
default here as well.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -255,4 +317,361 @@ EOP
is $dec->{one}[0], 1.0, "kind of dumb test";
}
+## A bytes/string value declares a length prefix; a malicious or truncated
+## input can declare far more bytes than actually exist. On a seekable reader
+## that is rejected before allocating for it.
+{
+ my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" }));
+
+ ## A length prefix declaring 100 MiB, with no payload following it.
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => 100 * 1024 * 1024,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader,
+ );
+ } qr/Cannot read/, "oversized bytes length is rejected before allocating";
+
+ ## A well-formed value above the check threshold whose data is present
+ ## still decodes.
+ my $payload = 'x' x (2 * 1024 * 1024);
+ my $enc2 = '';
+ Avro::BinaryEncoder->encode(
+ schema => $bytes_schema,
+ data => $payload,
+ emit_cb => sub { $enc2 .= ${ $_[0] } },
+ );
+ open my $reader2, '<', \$enc2 or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader2,
+ );
+ is $dec, $payload, "within-limit bytes value still decodes";
+}
+
+## An array/map block declares an element count; a malicious or truncated input
+## can declare far more elements than the remaining bytes could hold. The count
+## is validated against the bytes remaining before iterating, using the minimum
+## on-wire size of the element schema (so 0-byte elements like null are not
+## falsely rejected).
+sub encode_long {
+ my ($value) = @_;
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => $value,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ return $enc;
+}
+
+{
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"long" }));
+ my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized array count is rejected before
iterating";
+}
+
+{
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long"
}));
+ my $enc = encode_long(1_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map count is rejected before
iterating";
+}
+
+{
+ ## An array of nulls: null elements occupy zero bytes, so a large count is
+ ## legitimate and must not be rejected.
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"null" }));
+ my $count = 100_000;
+ my $enc = encode_long($count) . encode_long(0); # one block + end marker
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ is scalar(@$dec), $count, "array of nulls is not falsely rejected";
+}
+
+## Zero-byte elements (null, zero-length fixed, all-null records) consume no
+## input, so the bytes-remaining check cannot bound them. A huge declared block
+## count of such elements is capped instead.
+sub decode_zero_byte_array {
+ my ($items_schema, $enc) = @_;
+ my $schema = Avro::Schema->parse(qq({ "type": "array", "items":
$items_schema }));
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ return Avro::BinaryDecoder->decode(
+ writer_schema => $schema,
+ reader_schema => $schema,
+ reader => $reader,
+ );
+}
+
+{
+ ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is
+ ## rejected by the default limit, before allocating. Pin the limit to the
+ ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS
was
+ ## set in the environment when the module was loaded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS =
+ $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(200_000_000) .
encode_long(0));
+ } qr/more than \d+ zero-byte/, "array of 200M nulls rejected by default
limit";
+}
+
+{
+ ## Within a lowered limit, a legitimate small null array still decodes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ my $dec = decode_zero_byte_array('"null"', encode_long(1000) .
encode_long(0));
+ is scalar(@$dec), 1000, "array of nulls within configured limit still
reads";
+}
+
+{
+ ## INT64_MIN as a block count is the pathological negation case. Negating
it
+ ## yields 2**63, which the cap rejects. INT64_MIN zig-zag encodes as the
+ ## 10-byte varint below, followed by a block byte-size (0).
+ my $int64_min = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01";
+ throws_ok {
+ decode_zero_byte_array('"null"', $int64_min . encode_long(0));
+ } qr/Cannot read a collection|zero-byte|Invalid/, "INT64_MIN array block
count rejected";
+}
+
+{
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(1001) . encode_long(0));
+ } qr/more than 1000 zero-byte/, "array of nulls over configured limit
rejected";
+}
+
+{
+ ## Cumulative across blocks: two blocks of 600 (1200 > 1000).
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(600) . encode_long(600) .
encode_long(0));
+ } qr/more than 1000 zero-byte/, "cumulative null blocks over limit
rejected";
+}
+
+{
+ ## A negative block count (abs value + block byte-size) is normalized and
+ ## must still be bounded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(-200_000) .
encode_long(0));
+ } qr/more than 1000 zero-byte/, "negative null block count rejected";
+}
+
+{
+ ## A record whose only field is null encodes to zero bytes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('{ "type": "record", "name": "R", "fields":
[{"name":"n","type":"null"}] }',
+ encode_long(2000) . encode_long(0));
+ } qr/more than 1000 zero-byte/, "array of all-null records over limit
rejected";
+}
+
+{
+ ## A huge map<null> is bounded by the bytes-remaining check (each entry
has a
+ ## >= 1 byte key), not the zero-byte cap.
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "null"
}));
+ my $enc = encode_long(200_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map<null> rejected by available-bytes
check";
+}
+
+{
+ ## Skipping a huge array<null> writer field absent from the reader schema
is
+ ## bounded (a CPU exhaustion otherwise), via skip_block.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ my $w_schema = Avro::Schema->parse(<<'EOJ');
+ { "type": "record", "name": "test",
+ "fields" : [
+ {"name": "arr", "type": {"type": "array", "items": "null"}},
+ {"name": "a", "type": "long"} ]}
+EOJ
+ my $r_schema = Avro::Schema->parse(<<'EOJ');
+ { "type": "record", "name": "test",
+ "fields" : [ {"name": "a", "type": "long"} ]}
+EOJ
+ # Hand-crafted: array block of 2000 nulls + end marker, then a = 42.
+ my $enc = encode_long(2000) . encode_long(0) . encode_long(42);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $w_schema,
+ reader_schema => $r_schema,
+ reader => $reader,
+ );
+ } qr/more than 1000/, "skipping oversized array<null> field is bounded";
+}
+
+{
+ ## The zero-byte cap raises the dedicated exception class.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
Review Comment:
This exception-class assertion intends to exercise the zero-byte item cap,
but it only localizes `MAX_COLLECTION_ITEMS`. If `MAX_COLLECTION_STRUCTURAL`
was initialized to a smaller value via `AVRO_MAX_COLLECTION_ITEMS`, the
structural cap may fire first and this block would no longer be specifically
testing the zero-byte-cap path. Localize `MAX_COLLECTION_STRUCTURAL` to the
default here as well.
##########
lang/perl/t/03_bin_decode.t:
##########
@@ -255,4 +317,361 @@ EOP
is $dec->{one}[0], 1.0, "kind of dumb test";
}
+## A bytes/string value declares a length prefix; a malicious or truncated
+## input can declare far more bytes than actually exist. On a seekable reader
+## that is rejected before allocating for it.
+{
+ my $bytes_schema = Avro::Schema->parse(q({ "type": "bytes" }));
+
+ ## A length prefix declaring 100 MiB, with no payload following it.
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => 100 * 1024 * 1024,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader,
+ );
+ } qr/Cannot read/, "oversized bytes length is rejected before allocating";
+
+ ## A well-formed value above the check threshold whose data is present
+ ## still decodes.
+ my $payload = 'x' x (2 * 1024 * 1024);
+ my $enc2 = '';
+ Avro::BinaryEncoder->encode(
+ schema => $bytes_schema,
+ data => $payload,
+ emit_cb => sub { $enc2 .= ${ $_[0] } },
+ );
+ open my $reader2, '<', \$enc2 or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $bytes_schema,
+ reader_schema => $bytes_schema,
+ reader => $reader2,
+ );
+ is $dec, $payload, "within-limit bytes value still decodes";
+}
+
+## An array/map block declares an element count; a malicious or truncated input
+## can declare far more elements than the remaining bytes could hold. The count
+## is validated against the bytes remaining before iterating, using the minimum
+## on-wire size of the element schema (so 0-byte elements like null are not
+## falsely rejected).
+sub encode_long {
+ my ($value) = @_;
+ my $enc = '';
+ Avro::BinaryEncoder->encode(
+ schema => Avro::Schema->parse(q({ "type": "long" })),
+ data => $value,
+ emit_cb => sub { $enc .= ${ $_[0] } },
+ );
+ return $enc;
+}
+
+{
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"long" }));
+ my $enc = encode_long(1_000_000); # 1,000,000 longs, no element data
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized array count is rejected before
iterating";
+}
+
+{
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "long"
}));
+ my $enc = encode_long(1_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map count is rejected before
iterating";
+}
+
+{
+ ## An array of nulls: null elements occupy zero bytes, so a large count is
+ ## legitimate and must not be rejected.
+ my $array_schema = Avro::Schema->parse(q({ "type": "array", "items":
"null" }));
+ my $count = 100_000;
+ my $enc = encode_long($count) . encode_long(0); # one block + end marker
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ my $dec = Avro::BinaryDecoder->decode(
+ writer_schema => $array_schema,
+ reader_schema => $array_schema,
+ reader => $reader,
+ );
+ is scalar(@$dec), $count, "array of nulls is not falsely rejected";
+}
+
+## Zero-byte elements (null, zero-length fixed, all-null records) consume no
+## input, so the bytes-remaining check cannot bound them. A huge declared block
+## count of such elements is capped instead.
+sub decode_zero_byte_array {
+ my ($items_schema, $enc) = @_;
+ my $schema = Avro::Schema->parse(qq({ "type": "array", "items":
$items_schema }));
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ return Avro::BinaryDecoder->decode(
+ writer_schema => $schema,
+ reader_schema => $schema,
+ reader => $reader,
+ );
+}
+
+{
+ ## The reported exploit: a ~6 byte payload declaring 200,000,000 nulls is
+ ## rejected by the default limit, before allocating. Pin the limit to the
+ ## default so the test is deterministic even if AVRO_MAX_COLLECTION_ITEMS
was
+ ## set in the environment when the module was loaded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS =
+ $Avro::BinaryDecoder::DEFAULT_MAX_COLLECTION_ITEMS;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(200_000_000) .
encode_long(0));
+ } qr/more than \d+ zero-byte/, "array of 200M nulls rejected by default
limit";
+}
+
+{
+ ## Within a lowered limit, a legitimate small null array still decodes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ my $dec = decode_zero_byte_array('"null"', encode_long(1000) .
encode_long(0));
+ is scalar(@$dec), 1000, "array of nulls within configured limit still
reads";
+}
+
+{
+ ## INT64_MIN as a block count is the pathological negation case. Negating
it
+ ## yields 2**63, which the cap rejects. INT64_MIN zig-zag encodes as the
+ ## 10-byte varint below, followed by a block byte-size (0).
+ my $int64_min = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01";
+ throws_ok {
+ decode_zero_byte_array('"null"', $int64_min . encode_long(0));
+ } qr/Cannot read a collection|zero-byte|Invalid/, "INT64_MIN array block
count rejected";
+}
+
+{
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(1001) . encode_long(0));
+ } qr/more than 1000 zero-byte/, "array of nulls over configured limit
rejected";
+}
+
+{
+ ## Cumulative across blocks: two blocks of 600 (1200 > 1000).
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(600) . encode_long(600) .
encode_long(0));
+ } qr/more than 1000 zero-byte/, "cumulative null blocks over limit
rejected";
+}
+
+{
+ ## A negative block count (abs value + block byte-size) is normalized and
+ ## must still be bounded.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('"null"', encode_long(-200_000) .
encode_long(0));
+ } qr/more than 1000 zero-byte/, "negative null block count rejected";
+}
+
+{
+ ## A record whose only field is null encodes to zero bytes.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
+ throws_ok {
+ decode_zero_byte_array('{ "type": "record", "name": "R", "fields":
[{"name":"n","type":"null"}] }',
+ encode_long(2000) . encode_long(0));
+ } qr/more than 1000 zero-byte/, "array of all-null records over limit
rejected";
+}
+
+{
+ ## A huge map<null> is bounded by the bytes-remaining check (each entry
has a
+ ## >= 1 byte key), not the zero-byte cap.
+ my $map_schema = Avro::Schema->parse(q({ "type": "map", "values": "null"
}));
+ my $enc = encode_long(200_000_000);
+ open my $reader, '<', \$enc or die "Can't open memory file: $!";
+ throws_ok {
+ Avro::BinaryDecoder->decode(
+ writer_schema => $map_schema,
+ reader_schema => $map_schema,
+ reader => $reader,
+ );
+ } qr/Collection claims/, "oversized map<null> rejected by available-bytes
check";
+}
+
+{
+ ## Skipping a huge array<null> writer field absent from the reader schema
is
+ ## bounded (a CPU exhaustion otherwise), via skip_block.
+ local $Avro::BinaryDecoder::MAX_COLLECTION_ITEMS = 1000;
Review Comment:
This skip-bounding test localizes `MAX_COLLECTION_ITEMS` but not
`MAX_COLLECTION_STRUCTURAL`. If the env var initialized the structural cap to a
small value, it can override the intended test setup and change the thrown
message/limit, making the test dependent on the environment. Localize
`MAX_COLLECTION_STRUCTURAL` to the default here too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
