iemejia commented on code in PR #3852:
URL: https://github.com/apache/avro/pull/3852#discussion_r3564493769
##########
lang/js/lib/files.js:
##########
@@ -187,12 +208,40 @@ BlockDecoder.prototype._decodeHeader = function () {
}
var codec = (header.meta['avro.codec'] || 'null').toString();
- this._decompress = (this._codecs || BlockDecoder.getDefaultCodecs())[codec];
- if (!this._decompress) {
+ var decompress = (this._codecs || BlockDecoder.getDefaultCodecs())[codec];
+ if (!decompress) {
this.emit('error', new Error(f('unknown codec: %s', codec)));
return;
}
+ // Bound the decompressed size of each block to guard against a block with a
+ // very high compression ratio (or a malformed block) expanding to far more
+ // memory than its compressed size. For the built-in deflate codec the limit
+ // is passed to zlib so the allocation itself is capped; every codec's output
+ // is additionally checked as a safeguard.
+ var maxLength = this._maxDecompressLength;
+ if (!this._codecs && codec === 'deflate') {
+ decompress = function (buf, cb) {
+ zlib.inflateRaw(buf, {maxOutputLength: maxLength}, cb);
+ };
+ }
Review Comment:
Fixed in a4a66808c7 — the zlib maxOutputLength cap is now applied whenever
the resolved codec is zlib.inflateRaw, including a custom codecs map that
reuses it. Added a test for that case.
##########
lang/js/lib/files.js:
##########
@@ -187,12 +208,40 @@ BlockDecoder.prototype._decodeHeader = function () {
}
var codec = (header.meta['avro.codec'] || 'null').toString();
- this._decompress = (this._codecs || BlockDecoder.getDefaultCodecs())[codec];
- if (!this._decompress) {
+ var decompress = (this._codecs || BlockDecoder.getDefaultCodecs())[codec];
+ if (!decompress) {
this.emit('error', new Error(f('unknown codec: %s', codec)));
return;
}
+ // Bound the decompressed size of each block to guard against a block with a
+ // very high compression ratio (or a malformed block) expanding to far more
+ // memory than its compressed size. For the built-in deflate codec the limit
+ // is passed to zlib so the allocation itself is capped; every codec's output
+ // is additionally checked as a safeguard.
+ var maxLength = this._maxDecompressLength;
+ if (!this._codecs && codec === 'deflate') {
+ decompress = function (buf, cb) {
+ zlib.inflateRaw(buf, {maxOutputLength: maxLength}, cb);
+ };
+ }
+ this._decompress = function (buf, cb) {
+ decompress(buf, function (err, data) {
+ if (err) {
+ cb(err);
+ return;
+ }
+ if (data && data.length > maxLength) {
+ cb(new Error(f(
+ 'decompressed block size exceeds the maximum allowed of %d bytes',
+ maxLength
+ )));
+ return;
+ }
+ cb(null, data);
+ });
Review Comment:
Fixed in a4a66808c7 — _pending is now decremented in the block callback's
error path, so a failed decompression (including the size safeguard firing) can
no longer leave the stream waiting at finish.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
