Copilot commented on code in PR #3855:
URL: https://github.com/apache/avro/pull/3855#discussion_r3564681034
##########
lang/perl/lib/Avro/DataFileReader.pm:
##########
@@ -215,24 +220,98 @@ sub read_block_header {
my $marker = substr $block, -(MARKER_SIZE), MARKER_SIZE, '';
$datafile->{block_marker} = $marker;
+ ## The decompressed size of a block is capped to guard against a block with
+ ## a very high compression ratio expanding to far more memory than its
+ ## compressed size. The limit is the AVRO_MAX_DECOMPRESS_LENGTH environment
+ ## variable, or DEFAULT_MAX_DECOMPRESS_LENGTH. (Note: block_max_size is a
+ ## writer-side flush threshold measured in compressed bytes and is not
reused
+ ## here to avoid conflating the two units.)
+ my $limit = _max_decompress_length();
+
## this is our new reader
$datafile->{reader} = do {
if ($codec eq 'deflate') {
- IO::Uncompress::RawInflate->new(\$block);
+ my $z = IO::Uncompress::RawInflate->new(\$block)
+ or croak "Error inflating block:
$IO::Uncompress::RawInflate::RawInflateError";
+ my $uncompressed = _inflate_bounded($z, $limit);
+ do { open my $fh, '<', \$uncompressed; $fh };
}
elsif ($codec eq 'bzip2') {
- my $uncompressed;
- bunzip2 \$block => \$uncompressed;
- do { open $fh, '<', \$uncompressed; $fh };
+ my $z = IO::Uncompress::Bunzip2->new(\$block)
+ or croak "Error decompressing bzip2 block:
$IO::Uncompress::Bunzip2::Bunzip2Error";
+ my $uncompressed = _inflate_bounded($z, $limit);
+ do { open my $fh, '<', \$uncompressed; $fh };
}
elsif ($codec eq 'zstandard') {
- do { open $fh, '<', \(decompress(\$block)); $fh };
+ my $uncompressed = _zstd_decompress_bounded(\$block, $limit);
+ do { open my $fh, '<', \$uncompressed; $fh };
}
};
return;
}
+## Read from a streaming decompressor in chunks, rejecting the block as soon as
+## its decompressed size would exceed $limit so an over-large (or malicious)
+## block is not fully materialized in memory. Each read is sized to the
+## remaining budget (capped at 64 KiB) so the buffer overshoots $limit by at
+## most one byte before the check fires.
+sub _inflate_bounded {
+ my ($z, $limit) = @_;
+ my $uncompressed = '';
+ my $chunk;
+ my $status;
+ while (1) {
+ my $budget = $limit - length($uncompressed) + 1;
+ my $to_read = $budget < 65536 ? $budget : 65536;
+ $status = $z->read($chunk, $to_read);
+ last unless defined $status && $status > 0;
+ $uncompressed .= $chunk;
+ _check_decompress_length(length($uncompressed), $limit);
+ }
+ if (!defined $status || $status < 0) {
+ croak "Error decompressing block: " . $z->error;
+ }
+ return $uncompressed;
+}
+
+## Streaming zstandard decompression, bounded the same way as _inflate_bounded
so
+## a high-ratio block is rejected before its full form is materialized.
+sub _zstd_decompress_bounded {
+ my ($block_ref, $limit) = @_;
+ my $decompressor = Compress::Zstd::Decompressor->new;
+ my $uncompressed = '';
+ my $length = length($$block_ref);
+ my $offset = 0;
+ while ($offset < $length) {
+ my $piece = substr($$block_ref, $offset, 65536);
+ $offset += 65536;
+ my $out = $decompressor->decompress($piece);
+ next unless defined $out;
+ $uncompressed .= $out;
+ _check_decompress_length(length($uncompressed), $limit);
+ }
+ return $uncompressed;
Review Comment:
`_zstd_decompress_bounded` ignores `undef` returns from
`Compress::Zstd::Decompressor->decompress`, which the Compress::Zstd API uses
to signal either an error or that more input is needed. As written, a
decompression error can be silently skipped, and the function also never calls
`flush`, so valid streams that buffer output may be truncated. Handle the
`undef` case by checking `$decompressor->error` and, after feeding all input,
call `flush` and include its output (and enforce the size limit on it too).
##########
lang/perl/t/07_datafile_decompress_limit.t:
##########
@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#!/usr/bin/env perl
+
+# A data-file block is decompressed according to the file's codec. A block with
+# a very high compression ratio can expand to far more memory than its
+# compressed size. These tests ensure that reading such a block is rejected
+# instead of allocating without bound.
+
+use strict;
+use warnings;
+use File::Temp;
+use Avro::Schema;
+use Avro::DataFileWriter;
+use Test::More;
+use Test::Exception;
+
+use_ok 'Avro::DataFileReader';
+
+my $schema = Avro::Schema->parse('"string"');
+
+sub codec_file {
+ my ($codec, $payload) = @_;
+ my $fh = File::Temp->new(UNLINK => 1);
+ my $writer = Avro::DataFileWriter->new(
+ fh => $fh,
+ writer_schema => $schema,
+ codec => $codec,
+ );
+ $writer->print($payload);
+ $writer->flush;
+ seek $fh, 0, 0;
+ return $fh;
+}
+
+## A large, highly compressible value compresses to a tiny block but would
+## decompress to far more than the configured limit; reading it must be
rejected
+## for every codec. The limit is set via AVRO_MAX_DECOMPRESS_LENGTH.
+sub assert_codec_rejects_oversized {
+ my ($codec) = @_;
+ my $big = "a" x (64 * 1024); # 64 KiB
+ my $fh = codec_file($codec, $big);
+ local $ENV{AVRO_MAX_DECOMPRESS_LENGTH} = 1024;
+ my $reader = Avro::DataFileReader->new(
+ fh => $fh,
+ reader_schema => $schema,
+ );
+ throws_ok { $reader->all }
+ 'Avro::DataFile::Error::DecompressionSize',
+ "$codec block exceeding the limit is rejected";
+}
+
+assert_codec_rejects_oversized('deflate');
+
+SKIP: {
+ eval { require IO::Compress::Bzip2; 1 }
+ or skip 'IO::Compress::Bzip2 not available', 1;
+ assert_codec_rejects_oversized('bzip2');
+}
+
+SKIP: {
+ eval { require Compress::Zstd; 1 }
+ or skip 'Compress::Zstd not available', 1;
+ assert_codec_rejects_oversized('zstandard');
+}
+
+## A block within the limit still decodes correctly.
+{
+ my $payload = "hello world";
+ my $fh = codec_file('deflate', $payload);
+ local $ENV{AVRO_MAX_DECOMPRESS_LENGTH} = 1024 * 1024;
+ my $reader = Avro::DataFileReader->new(
+ fh => $fh,
+ reader_schema => $schema,
+ );
+ my @all = $reader->all;
+ is_deeply \@all, [$payload], 'deflate block within the limit decodes';
+}
+
+done_testing;
Review Comment:
The within-limit decode test only covers `deflate`, but this PR introduces
new bounded decompression paths for `bzip2` and `zstandard` as well. Adding
within-limit decoding assertions for those codecs (skipped when the codec
modules are unavailable) would better protect against regressions like missing
flush/error handling in the streaming paths.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
