Repository: hive Updated Branches: refs/heads/master fa4af2f17 -> b9e36a4aa
HIVE-18616 : work around HADOOP-15171 p2 (Sergey Shelukhin, reviewed by Gopal Vijayaraghavan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b9e36a4a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b9e36a4a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b9e36a4a Branch: refs/heads/master Commit: b9e36a4aa5141127704cccbbfbb586e0619a49da Parents: fa4af2f Author: sergey <[email protected]> Authored: Mon Feb 5 13:39:44 2018 -0800 Committer: sergey <[email protected]> Committed: Mon Feb 5 13:39:44 2018 -0800 ---------------------------------------------------------------------- .../ql/io/orc/encoded/EncodedReaderImpl.java | 48 +++----------------- 1 file changed, 6 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/b9e36a4a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java index 2bf8614..32bdf6e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamD import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.orc.CompressionCodec; +import org.apache.orc.CompressionKind; import org.apache.orc.DataReader; import org.apache.orc.OrcConf; import org.apache.orc.OrcFile.WriterVersion; @@ -1242,6 +1243,7 @@ class EncodedReaderImpl implements EncodedReader { LOG.trace("Decompressing " + src.remaining() + " bytes to dest buffer pos " + dest.position() + ", limit " + dest.limit()); } + codec.reset(); // We always need to call reset on the codec. codec.decompress(src, dest); dest.position(startPos); int newLim = dest.limit(); @@ -1249,49 +1251,11 @@ class EncodedReaderImpl implements EncodedReader { throw new AssertionError("After codec, buffer [" + startPos + ", " + startLim + ") became [" + dest.position() + ", " + newLim + ")"); } - if (dest.remaining() > 0) return; - - // There's a bug in native decompressor. See HADOOP-15171 - dest.limit(startLim); - src.position(startSrcPos); - src.limit(startSrcLim); - LOG.warn("The codec has produced 0 bytes for " + src.remaining() + " bytes at pos " - + src.position() + ", data hash " + src.hashCode() + ": [" + logSomeBytes(src)); - ByteBuffer srcHeap = ByteBuffer.allocate(src.remaining()), - destHeap = ByteBuffer.allocate(dest.remaining()); - int destHeapPos = destHeap.position(); - srcHeap.put(src); - srcHeap.position(startSrcPos); - codec.decompress(srcHeap, destHeap); - destHeap.position(destHeapPos); - int newLen = destHeap.remaining(); - LOG.warn("Fell back to JDK decompressor with memcopy; got " + newLen + " bytes"); - dest.put(destHeap); - dest.position(startPos); - dest.limit(startPos + newLen); - } - - private static String logSomeBytes(ByteBuffer src) { - final int max = 500; - StringBuilder sb = new StringBuilder(); - int base = src.position(), end = base + Math.min(max, src.remaining()); - for (int i = base; i < end; ++i) { - if (i != base) { - sb.append(' '); - } - int b = src.get(i) & 0xff; - if (b <= 0xf) { - sb.append('0'); - } - sb.append(Integer.toHexString(b)); - } - int rem = src.remaining() - max; - if (rem > 0) { - sb.append(" ... (").append(rem).append(" bytes)]"); - } else { - sb.append("]"); + if (dest.remaining() == 0) { + throw new IOException("The codec has produced 0 bytes for {" + src.isDirect() + ", " + + src.position() + ", " + src.remaining() + "} into {" + dest.isDirect() + ", " + + dest.position() + ", " + dest.remaining() + "}"); } - return sb.toString(); } private void ponderReleaseInitialRefcount(
