iemejia commented on code in PR #3853:
URL: https://github.com/apache/avro/pull/3853#discussion_r3564494195
##########
lang/c/src/codec.c:
##########
@@ -147,6 +173,12 @@ static int decode_snappy(avro_codec_t c, void * data,
int64_t len)
return 1;
}
Review Comment:
Fixed in 3aeb3265f8 — decode_snappy rejects a block shorter than the 4-byte
CRC suffix before computing len - 4, and the decompression limit is cached in a
local instead of being re-parsed in the error path.
##########
lang/c/src/codec.c:
##########
@@ -339,13 +381,24 @@ static int decode_deflate(avro_codec_t c, void * data,
int64_t len)
return 1;
}
- // The buffer was not big enough. resize it.
+ // The buffer was not big enough. resize it, without growing
beyond
+ // the configured maximum decompressed size.
if (err == Z_BUF_ERROR)
{
- c->block_data = avro_realloc(c->block_data,
c->block_size, c->block_size * 2);
+ int64_t new_size = c->block_size * 2;
+ if (new_size > max_len) {
+ new_size = max_len;
+ }
+ if (new_size <= c->block_size) {
+ inflateEnd(s);
+ avro_set_error("Decompressed block size exceeds
the maximum allowed of %lld bytes",
+ (long long) max_len);
+ return 1;
+ }
+ c->block_data = avro_realloc(c->block_data,
c->block_size, new_size);
s->next_out = c->block_data + s->total_out;
- s->avail_out += c->block_size;
- c->block_size = c->block_size * 2;
+ s->avail_out += (uInt)(new_size - c->block_size);
+ c->block_size = new_size;
Review Comment:
Fixed in 3aeb3265f8 — decode_deflate reallocs into a temporary pointer and
fails cleanly (freeing nothing it cannot) on OOM instead of null-dereferencing
and losing the original buffer.
##########
lang/c/src/codec.c:
##########
@@ -483,11 +537,28 @@ static int decode_lzma(avro_codec_t codec, void * data,
int64_t len)
codec->used_size = write_pos;
- // If it ran out of space to decode, give it more!!
- // It will continue where it left off because of read_pos and
write_pos.
+ // Reject a block that decompresses to more than the allowed
maximum,
+ // to guard against unbounded allocation from a high-ratio
block.
+ if ((int64_t) write_pos > max_len) {
+ avro_set_error("Decompressed block size exceeds the
maximum allowed of %lld bytes",
+ (long long) max_len);
+ return 1;
+ }
+
+ // If it ran out of space to decode, give it more (without
growing
+ // beyond the configured maximum decompressed size).
if (ret == LZMA_BUF_ERROR) {
- codec->block_data = avro_realloc(codec->block_data,
codec->block_size, codec->block_size * 2);
- codec->block_size = codec->block_size * 2;
+ int64_t new_size = codec->block_size * 2;
+ if (new_size > max_len) {
+ new_size = max_len;
+ }
+ if (new_size <= codec->block_size) {
+ avro_set_error("Decompressed block size exceeds
the maximum allowed of %lld bytes",
+ (long long) max_len);
+ return 1;
+ }
+ codec->block_data = avro_realloc(codec->block_data,
codec->block_size, new_size);
+ codec->block_size = new_size;
Review Comment:
Fixed in 3aeb3265f8 — decode_lzma applies the same temporary-pointer +
explicit OOM error pattern.
##########
lang/c/tests/test_avro_4284.c:
##########
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+/*
+ * AVRO-4284: a data-file block is decompressed according to the file's codec.
+ * A block with a very high compression ratio (or a malformed block) can expand
+ * to far more memory than its compressed size. Decompression must reject a
+ * block whose decompressed size would exceed the configured maximum, which
+ * these tests set to a small value via AVRO_MAX_DECOMPRESS_LENGTH.
+ */
+
+#include <avro.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "codec.h"
+
+/* Size of the highly compressible payload used to build an over-large block.
*/
+#define PAYLOAD_SIZE (4 * 1024 * 1024) /* 4 MiB of zeros */
+/* Decompression limit for the test, smaller than the payload. */
+#define TEST_LIMIT "1048576" /* 1 MiB */
+
+/*
+ * Compress `payload` with the named codec and attempt to decompress it with a
+ * decompression limit smaller than the payload. Returns 0 if the codec is not
+ * available (test skipped for that codec), 1 on test failure, 2 on success.
+ */
+static int
+check_codec_rejects_oversized(const char *name, const char *payload, int64_t
payload_len)
+{
+ struct avro_codec_t_ codec;
+ memset(&codec, 0, sizeof(codec));
+
+ if (avro_codec(&codec, name) != 0) {
+ fprintf(stderr, " codec %s not available, skipping\n", name);
+ return 0;
+ }
+
+ if (avro_codec_encode(&codec, (void *) payload, payload_len) != 0) {
+ fprintf(stderr, " codec %s: encode failed: %s\n", name,
avro_strerror());
+ avro_codec_reset(&codec);
+ return 1;
+ }
+
+ /* Copy the compressed bytes; decode reuses the codec's block buffer. */
+ int64_t compressed_len = codec.used_size;
+ char *compressed = (char *) malloc(compressed_len);
+ if (compressed == NULL) {
+ avro_codec_reset(&codec);
+ return 1;
+ }
+ memcpy(compressed, codec.block_data, compressed_len);
Review Comment:
Fixed in 3aeb3265f8 — the test casts used_size to size_t for malloc/memcpy
and guards against a non-positive length.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
