iemejia commented on code in PR #3850:
URL: https://github.com/apache/avro/pull/3850#discussion_r3568059464
##########
lang/py/avro/codecs.py:
##########
@@ -41,6 +42,55 @@
#
STRUCT_CRC32 = struct.Struct(">I") # big-endian unsigned int
+# Name of the environment variable used to override the default maximum size of
+# a single decompressed data-file block.
+MAX_DECOMPRESS_LENGTH_ENV = "AVRO_MAX_DECOMPRESS_LENGTH"
+
+# Default upper bound, in bytes, on the size a single data-file block may
+# decompress to. A block with a very high compression ratio (or a malformed
+# block) can otherwise expand to far more memory than its compressed size.
+# Reading a block that would decompress beyond this limit raises an
+# :class:`avro.errors.AvroDecompressionSizeException`. This mirrors the Java
+# SDK's ``org.apache.avro.limits.decompress.maxLength`` limit (AVRO-4247). The
+# default may be overridden with the ``AVRO_MAX_DECOMPRESS_LENGTH`` environment
+# variable.
+DEFAULT_MAX_DECOMPRESS_LENGTH = 200 * 1024 * 1024 # 200 MiB
+
+
+def _max_decompress_length() -> int:
+ """Return the maximum decompressed block size, honoring the environment
override."""
+ value = os.environ.get(MAX_DECOMPRESS_LENGTH_ENV)
+ if value is None:
+ return DEFAULT_MAX_DECOMPRESS_LENGTH
+ try:
+ parsed = int(value)
+ except ValueError:
+ return DEFAULT_MAX_DECOMPRESS_LENGTH
+ return parsed if parsed > 0 else DEFAULT_MAX_DECOMPRESS_LENGTH
Review Comment:
Fixed — _max_decompress_length() now returns `min(parsed, sys.maxsize)` so
an oversized override stays a valid Py_ssize_t and won't raise OverflowError
inside zlib/bz2 decompress().
##########
lang/py/avro/codecs.py:
##########
@@ -123,7 +173,22 @@ def decompress(readers_decoder: avro.io.BinaryDecoder) ->
avro.io.BinaryDecoder:
data = readers_decoder.read_bytes()
# -15 is the log of the window size; negative indicates
# "raw" (no zlib headers) decompression. See zlib.h.
- uncompressed = zlib.decompress(data, -15)
+ limit = _max_decompress_length()
+ decompressor = zlib.decompressobj(-15)
+ # Request at most limit + 1 bytes so that an over-large block is
detected
+ # without allocating the whole (potentially huge) output. Accumulate
into
+ # a bytearray so the flush() output is appended in place rather than
+ # creating an extra full-size copy of the already-decompressed data.
+ uncompressed = bytearray(decompressor.decompress(data, limit + 1))
+ if len(uncompressed) > limit:
+ _raise_decompression_too_large(limit)
+ uncompressed += decompressor.flush()
+ if len(uncompressed) > limit:
+ _raise_decompression_too_large(limit)
+ if not decompressor.eof:
+ # The end-of-stream marker was not reached: the block is truncated
or
+ # corrupt. zlib.decompress() used to raise for this; preserve that.
+ raise avro.errors.InvalidAvroBinaryEncoding("Truncated or corrupt
deflate block")
Review Comment:
Already bounded — the final flush is `decompressor.flush(limit + 1 -
len(uncompressed))` (codecs.py:194) followed by the size check, so flush()
can't materialize past the cap. The eof check runs after.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
