Copilot commented on code in PR #3850:
URL: https://github.com/apache/avro/pull/3850#discussion_r3567608042
##########
lang/py/avro/codecs.py:
##########
@@ -41,6 +42,55 @@
#
STRUCT_CRC32 = struct.Struct(">I") # big-endian unsigned int
+# Name of the environment variable used to override the default maximum size of
+# a single decompressed data-file block.
+MAX_DECOMPRESS_LENGTH_ENV = "AVRO_MAX_DECOMPRESS_LENGTH"
+
+# Default upper bound, in bytes, on the size a single data-file block may
+# decompress to. A block with a very high compression ratio (or a malformed
+# block) can otherwise expand to far more memory than its compressed size.
+# Reading a block that would decompress beyond this limit raises an
+# :class:`avro.errors.AvroDecompressionSizeException`. This mirrors the Java
+# SDK's ``org.apache.avro.limits.decompress.maxLength`` limit (AVRO-4247). The
+# default may be overridden with the ``AVRO_MAX_DECOMPRESS_LENGTH`` environment
+# variable.
+DEFAULT_MAX_DECOMPRESS_LENGTH = 200 * 1024 * 1024 # 200 MiB
+
+
+def _max_decompress_length() -> int:
+ """Return the maximum decompressed block size, honoring the environment
override."""
+ value = os.environ.get(MAX_DECOMPRESS_LENGTH_ENV)
+ if value is None:
+ return DEFAULT_MAX_DECOMPRESS_LENGTH
+ try:
+ parsed = int(value)
+ except ValueError:
+ return DEFAULT_MAX_DECOMPRESS_LENGTH
+ return parsed if parsed > 0 else DEFAULT_MAX_DECOMPRESS_LENGTH
Review Comment:
_max_decompress_length() returns the raw int parsed from
AVRO_MAX_DECOMPRESS_LENGTH without clamping. If a user sets this to an
extremely large value, downstream calls like zlib.decompress(...,
max_length=want) / bz2.BZ2Decompressor.decompress(..., max_length=want) can
raise OverflowError when converting to a Py_ssize_t, producing a confusing
exception instead of cleanly honoring the override (or falling back to the
default). Consider clamping to sys.maxsize (and treating non-positive values as
default) to keep behavior predictable.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
