Repository: hive Updated Branches: refs/heads/master 4fcf3d720 -> bac1d98c5
HIVE-20165: Enable ZLIB for streaming ingest (Prasanth Jayachandran, via Gopal V) Signed-off-by: Gopal V <gop...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bac1d98c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bac1d98c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bac1d98c Branch: refs/heads/master Commit: bac1d98c5e91cdb39567f21b2068e7951a93ef44 Parents: 4fcf3d7 Author: Prasanth Jayachandran <prasan...@apache.org> Authored: Tue Jul 17 20:17:55 2018 -0700 Committer: Gopal V <gop...@apache.org> Committed: Tue Jul 17 20:17:55 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 1 - .../apache/hive/streaming/TestStreaming.java | 21 ++++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/bac1d98c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java index 5590470..98f5df1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java @@ -342,7 +342,6 @@ public class OrcRecordUpdater implements RecordUpdater { writerOptions.blockPadding(false); if (optionsCloneForDelta.getConfiguration().getBoolean( HiveConf.ConfVars.HIVE_ORC_DELTA_STREAMING_OPTIMIZATIONS_ENABLED.varname, false)) { - writerOptions.compress(CompressionKind.NONE); writerOptions.encodingStrategy(org.apache.orc.OrcFile.EncodingStrategy.SPEED); writerOptions.rowIndexStride(0); writerOptions.getConfiguration().set(OrcConf.DICTIONARY_KEY_SIZE_THRESHOLD.getAttribute(), "-1.0"); http://git-wip-us.apache.org/repos/asf/hive/blob/bac1d98c/streaming/src/test/org/apache/hive/streaming/TestStreaming.java ---------------------------------------------------------------------- diff --git a/streaming/src/test/org/apache/hive/streaming/TestStreaming.java b/streaming/src/test/org/apache/hive/streaming/TestStreaming.java index 1f05d88..8b5e508 100644 --- a/streaming/src/test/org/apache/hive/streaming/TestStreaming.java +++ b/streaming/src/test/org/apache/hive/streaming/TestStreaming.java @@ -2072,20 +2072,19 @@ public class TestStreaming { System.setOut(origOut); String outDump = new String(myOut.toByteArray()); - // make sure delta files are written with no indexes, no compression and no dictionary - // no compression - Assert.assertEquals(true, outDump.contains("Compression: NONE")); + // make sure delta files are written with no indexes and no dictionary + Assert.assertEquals(true, outDump.contains("Compression: ZLIB")); // no stats/indexes Assert.assertEquals(true, outDump.contains("Column 0: count: 0 hasNull: false")); - Assert.assertEquals(true, outDump.contains("Column 1: count: 0 hasNull: false bytesOnDisk: 12 sum: 0")); - Assert.assertEquals(true, outDump.contains("Column 2: count: 0 hasNull: false bytesOnDisk: 12 sum: 0")); - Assert.assertEquals(true, outDump.contains("Column 3: count: 0 hasNull: false bytesOnDisk: 24 sum: 0")); - Assert.assertEquals(true, outDump.contains("Column 4: count: 0 hasNull: false bytesOnDisk: 14 sum: 0")); - Assert.assertEquals(true, outDump.contains("Column 5: count: 0 hasNull: false bytesOnDisk: 12 sum: 0")); + Assert.assertEquals(true, outDump.contains("Column 1: count: 0 hasNull: false bytesOnDisk: 15 sum: 0")); + Assert.assertEquals(true, outDump.contains("Column 2: count: 0 hasNull: false bytesOnDisk: 15 sum: 0")); + Assert.assertEquals(true, outDump.contains("Column 3: count: 0 hasNull: false bytesOnDisk: 19 sum: 0")); + Assert.assertEquals(true, outDump.contains("Column 4: count: 0 hasNull: false bytesOnDisk: 17 sum: 0")); + Assert.assertEquals(true, outDump.contains("Column 5: count: 0 hasNull: false bytesOnDisk: 15 sum: 0")); Assert.assertEquals(true, outDump.contains("Column 6: count: 0 hasNull: false")); - Assert.assertEquals(true, outDump.contains("Column 7: count: 0 hasNull: false bytesOnDisk: 11864")); - Assert.assertEquals(true, outDump.contains("Column 8: count: 0 hasNull: false bytesOnDisk: 2033 sum: 0")); - Assert.assertEquals(true, outDump.contains("Column 9: count: 0 hasNull: false bytesOnDisk: 13629")); + Assert.assertEquals(true, outDump.contains("Column 7: count: 0 hasNull: false bytesOnDisk: 3929")); + Assert.assertEquals(true, outDump.contains("Column 8: count: 0 hasNull: false bytesOnDisk: 1484 sum: 0")); + Assert.assertEquals(true, outDump.contains("Column 9: count: 0 hasNull: false bytesOnDisk: 816")); // no dictionary Assert.assertEquals(true, outDump.contains("Encoding column 7: DIRECT_V2")); Assert.assertEquals(true, outDump.contains("Encoding column 9: DIRECT_V2"));
