HADOOP-15822. zstd compressor can fail with a small output buffer. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e39f4f5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e39f4f5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e39f4f5 Branch: refs/heads/HDFS-13891 Commit: 3e39f4f56e74eaa9646a8f3cf5a04effaec571ec Parents: 577ef1b Author: Akira Ajisaka <aajis...@apache.org> Authored: Wed Oct 24 13:49:28 2018 +0900 Committer: Brahma Reddy Battula <bra...@apache.org> Committed: Tue Oct 30 11:31:17 2018 +0530 ---------------------------------------------------------------------- dev-support/docker/Dockerfile | 1 + .../hadoop/io/compress/zstd/ZStandardDecompressor.java | 4 ++-- .../apache/hadoop/io/compress/zstd/ZStandardCompressor.c | 11 +++++++---- .../hadoop/io/compress/zstd/ZStandardDecompressor.c | 1 + .../zstd/TestZStandardCompressorDecompressor.java | 10 ++++------ 5 files changed, 15 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e39f4f5/dev-support/docker/Dockerfile ---------------------------------------------------------------------- diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 7679500..67140c9 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -63,6 +63,7 @@ RUN apt-get -q update \ libsnappy-dev \ libssl-dev \ libtool \ + libzstd1-dev \ locales \ make \ pinentry-curses \ http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e39f4f5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java index 73d73e1..bc9d29c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java @@ -262,8 +262,8 @@ public class ZStandardDecompressor implements Decompressor { int originalPosition = dst.position(); int n = inflateBytesDirect( - src, src.position(), src.remaining(), dst, dst.position(), - dst.remaining() + src, src.position(), src.limit(), dst, dst.position(), + dst.limit() ); dst.position(originalPosition + n); if (bytesInCompressedBuffer > 0) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e39f4f5/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c index 055683a..289554b 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c @@ -195,10 +195,13 @@ JNIEXPORT jint Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_defla ZSTD_inBuffer input = { uncompressed_bytes, uncompressed_direct_buf_len, uncompressed_direct_buf_off }; ZSTD_outBuffer output = { compressed_bytes, compressed_direct_buf_len, 0 }; - size_t size = dlsym_ZSTD_compressStream(stream, &output, &input); - if (dlsym_ZSTD_isError(size)) { - THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(size)); - return (jint) 0; + size_t size; + if (uncompressed_direct_buf_len != 0) { + size = dlsym_ZSTD_compressStream(stream, &output, &input); + if (dlsym_ZSTD_isError(size)) { + THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(size)); + return (jint) 0; + } } if (finish && input.pos == input.size) { // end the stream, flush and write the frame epilogue http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e39f4f5/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c index b97dd17..e28359b 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c @@ -178,6 +178,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompre return (jint) 0; } uncompressed_bytes = ((char*) uncompressed_bytes) + uncompressed_direct_buf_off; + uncompressed_direct_buf_len -= uncompressed_direct_buf_off; ZSTD_inBuffer input = { compressed_bytes, compressed_direct_buf_len, compressed_direct_buf_off }; ZSTD_outBuffer output = { uncompressed_bytes, uncompressed_direct_buf_len, 0 }; http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e39f4f5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java index 04def24..dcfb7e9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java @@ -414,13 +414,11 @@ public class TestZStandardCompressorDecompressor { outBuf.clear(); while (!decompressor.finished()) { decompressor.decompress(inBuf, outBuf); - if (outBuf.remaining() == 0) { - outBuf.flip(); - while (outBuf.remaining() > 0) { - assertEquals(expected.get(), outBuf.get()); - } - outBuf.clear(); + outBuf.flip(); + while (outBuf.remaining() > 0) { + assertEquals(expected.get(), outBuf.get()); } + outBuf.clear(); } outBuf.flip(); while (outBuf.remaining() > 0) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
