Repository: hive Updated Branches: refs/heads/master 8fa9d5833 -> b4d1f96c6
HIVE-15841: Upgrade Hive to ORC 1.3.3 (Owen O'Malley reviewed by Prasanth Jayachandran) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b4d1f96c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b4d1f96c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b4d1f96c Branch: refs/heads/master Commit: b4d1f96c6f7a1652af94c1d8202f214ed27f87d1 Parents: 8fa9d58 Author: Prasanth Jayachandran <prasan...@apache.org> Authored: Thu Mar 23 22:07:35 2017 -0700 Committer: Prasanth Jayachandran <prasan...@apache.org> Committed: Thu Mar 23 22:07:35 2017 -0700 ---------------------------------------------------------------------- .../llap/io/encoded/OrcEncodedDataReader.java | 6 +- .../llap/io/encoded/SerDeEncodedDataReader.java | 2 +- pom.xml | 2 +- .../apache/hadoop/hive/ql/io/orc/OrcFile.java | 4 +- .../hadoop/hive/ql/io/orc/OrcInputFormat.java | 2 +- .../hadoop/hive/ql/io/orc/TestOrcFile.java | 17 +-- .../hive/ql/io/orc/TestOrcRawRecordMerger.java | 7 +- .../clientpositive/llap/orc_analyze.q.out | 32 +++--- .../clientpositive/llap/orc_merge10.q.out | 4 +- .../clientpositive/llap/orc_merge11.q.out | 110 +++++++++---------- .../clientpositive/llap/orc_merge12.q.out | 4 +- .../clientpositive/llap/union_fast_stats.q.out | 12 +- .../results/clientpositive/orc_file_dump.q.out | 110 +++++++++---------- .../results/clientpositive/orc_merge10.q.out | 4 +- .../results/clientpositive/orc_merge11.q.out | 110 +++++++++---------- .../results/clientpositive/orc_merge12.q.out | 4 +- .../spark/vector_outer_join1.q.out | 40 +++---- .../spark/vector_outer_join2.q.out | 16 +-- .../clientpositive/tez/orc_merge12.q.out | 4 +- 19 files changed, 238 insertions(+), 252 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java index ce0ffe6..076b0e1 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java @@ -696,10 +696,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void> ArrayList<OrcStripeMetadata> metadata) throws IOException { RecordReaderImpl.SargApplier sargApp = null; if (sarg != null && rowIndexStride != 0) { - List<OrcProto.Type> types = fileMetadata.getTypes(); - String[] colNamesForSarg = OrcInputFormat.getSargColumnNames( - columnNames, types, globalIncludes, fileMetadata.isOriginalFormat()); - sargApp = new RecordReaderImpl.SargApplier(sarg, colNamesForSarg, + sargApp = new RecordReaderImpl.SargApplier(sarg, rowIndexStride, evolution, OrcFile.WriterVersion.from(fileMetadata.getWriterVersionNum())); } @@ -714,6 +711,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void> OrcStripeMetadata stripeMetadata = metadata.get(stripeIxMod); rgsToRead = sargApp.pickRowGroups(stripe, stripeMetadata.getRowIndexes(), stripeMetadata.getBloomFilterKinds(), + stripeMetadata.getEncodings(), stripeMetadata.getBloomFilterIndexes(), true); } boolean isNone = rgsToRead == RecordReaderImpl.SargApplier.READ_NO_RGS, http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java index 6aab6de..a4fde6d 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java @@ -218,7 +218,7 @@ public class SerDeEncodedDataReader extends CallableWithNdc<Void> this.jobConf = jobConf; this.schema = schema; this.writerIncludes = OrcInputFormat.genIncludedColumns(schema, columnIds); - SchemaEvolution evolution = new SchemaEvolution(schema, + SchemaEvolution evolution = new SchemaEvolution(schema, null, new Reader.Options(jobConf).include(writerIncludes)); consumer.setSchemaEvolution(evolution); } http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 3ea3c77..fc7aa44 100644 --- a/pom.xml +++ b/pom.xml @@ -175,7 +175,7 @@ <libthrift.version>0.9.3</libthrift.version> <log4j2.version>2.6.2</log4j2.version> <opencsv.version>2.3</opencsv.version> - <orc.version>1.3.1</orc.version> + <orc.version>1.3.3</orc.version> <mockito-all.version>1.9.5</mockito-all.version> <mina.version>2.0.0-M5</mina.version> <netty.version>4.0.29.Final</netty.version> http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java index 96ca736..cbd38ed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.orc.FileMetadata; import org.apache.orc.PhysicalWriter; -import org.apache.orc.impl.MemoryManager; +import org.apache.orc.MemoryManager; import org.apache.orc.TypeDescription; import org.apache.orc.impl.OrcTail; @@ -258,7 +258,7 @@ public final class OrcFile extends org.apache.orc.OrcFile { /** * A package local option to set the memory manager. */ - protected WriterOptions memory(MemoryManager value) { + public WriterOptions memory(MemoryManager value) { super.memory(value); return this; } http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java index 8318a62..8fb7211 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java @@ -1531,7 +1531,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>, Reader.Options readerOptions = new Reader.Options(context.conf); if (readerTypes == null) { readerIncluded = genIncludedColumns(fileSchema, context.conf); - evolution = new SchemaEvolution(fileSchema, readerOptions.include(readerIncluded)); + evolution = new SchemaEvolution(fileSchema, null, readerOptions.include(readerIncluded)); } else { // The reader schema always comes in without ACID columns. TypeDescription readerSchema = OrcUtils.convertTypeFromProtobuf(readerTypes, 0); http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java index 84e83df..dae2a5f 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java @@ -44,8 +44,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.llap.TypeDesc; import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory; @@ -86,7 +84,7 @@ import org.apache.orc.DecimalColumnStatistics; import org.apache.orc.DoubleColumnStatistics; import org.apache.orc.IntegerColumnStatistics; import org.apache.orc.OrcConf; -import org.apache.orc.impl.MemoryManager; +import org.apache.orc.MemoryManager; import org.apache.orc.OrcProto; import org.apache.orc.OrcUtils; @@ -1924,7 +1922,7 @@ public class TestOrcFile { new MiddleStruct(inner, inner2), list(), map(inner,inner2)); } - private static class MyMemoryManager extends MemoryManager { + private static class MyMemoryManager implements MemoryManager { final long totalSpace; double rate; Path path = null; @@ -1933,7 +1931,6 @@ public class TestOrcFile { MemoryManager.Callback callback; MyMemoryManager(Configuration conf, long totalSpace, double rate) { - super(conf); this.totalSpace = totalSpace; this.rate = rate; } @@ -1953,16 +1950,6 @@ public class TestOrcFile { } @Override - public long getTotalMemoryPool() { - return totalSpace; - } - - @Override - public double getAllocationScale() { - return rate; - } - - @Override public void addedRow(int count) throws IOException { rows += count; if (rows >= 100) { http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java index f07aa49..1ce1bfb 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java @@ -19,8 +19,9 @@ package org.apache.hadoop.hive.ql.io.orc; import org.apache.orc.CompressionKind; -import org.apache.orc.impl.MemoryManager; +import org.apache.orc.MemoryManager; import org.apache.orc.StripeInformation; +import org.apache.orc.impl.MemoryManagerImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -852,7 +853,7 @@ public class TestOrcRawRecordMerger { } // write the base - MemoryManager mgr = new MemoryManager(conf){ + MemoryManager mgr = new MemoryManagerImpl(conf){ int rowsAddedSinceCheck = 0; @Override @@ -955,7 +956,7 @@ public class TestOrcRawRecordMerger { } // write the base - MemoryManager mgr = new MemoryManager(conf){ + MemoryManager mgr = new MemoryManagerImpl(conf){ int rowsAddedSinceCheck = 0; @Override http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/ql/src/test/results/clientpositive/llap/orc_analyze.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/orc_analyze.q.out b/ql/src/test/results/clientpositive/llap/orc_analyze.q.out index fdfb74a..1cc9d61 100644 --- a/ql/src/test/results/clientpositive/llap/orc_analyze.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_analyze.q.out @@ -102,7 +102,7 @@ Table Parameters: numFiles 1 numRows 100 rawDataSize 52600 - totalSize 3197 + totalSize 3200 #### A masked pattern was here #### # Storage Information @@ -150,7 +150,7 @@ Table Parameters: numFiles 1 numRows 100 rawDataSize 52600 - totalSize 3197 + totalSize 3200 #### A masked pattern was here #### # Storage Information @@ -198,7 +198,7 @@ Table Parameters: numFiles 1 numRows 100 rawDataSize 52600 - totalSize 3197 + totalSize 3200 #### A masked pattern was here #### # Storage Information @@ -285,7 +285,7 @@ Table Parameters: numFiles 1 numRows 100 rawDataSize 52600 - totalSize 3197 + totalSize 3200 #### A masked pattern was here #### # Storage Information @@ -438,7 +438,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22050 - totalSize 2113 + totalSize 2114 #### A masked pattern was here #### # Storage Information @@ -536,7 +536,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22050 - totalSize 2113 + totalSize 2114 #### A masked pattern was here #### # Storage Information @@ -634,7 +634,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22050 - totalSize 2113 + totalSize 2114 #### A masked pattern was here #### # Storage Information @@ -775,7 +775,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22050 - totalSize 2113 + totalSize 2114 #### A masked pattern was here #### # Storage Information @@ -891,7 +891,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 21975 - totalSize 5259 + totalSize 5260 #### A masked pattern was here #### # Storage Information @@ -934,7 +934,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22043 - totalSize 5326 + totalSize 5331 #### A masked pattern was here #### # Storage Information @@ -989,7 +989,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 21975 - totalSize 5259 + totalSize 5260 #### A masked pattern was here #### # Storage Information @@ -1032,7 +1032,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22043 - totalSize 5326 + totalSize 5331 #### A masked pattern was here #### # Storage Information @@ -1087,7 +1087,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 21975 - totalSize 5259 + totalSize 5260 #### A masked pattern was here #### # Storage Information @@ -1130,7 +1130,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22043 - totalSize 5326 + totalSize 5331 #### A masked pattern was here #### # Storage Information @@ -1234,7 +1234,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 21975 - totalSize 5259 + totalSize 5260 #### A masked pattern was here #### # Storage Information @@ -1277,7 +1277,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22043 - totalSize 5326 + totalSize 5331 #### A masked pattern was here #### # Storage Information http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/ql/src/test/results/clientpositive/llap/orc_merge10.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/orc_merge10.q.out b/ql/src/test/results/clientpositive/llap/orc_merge10.q.out index 0197592..dd5d1cb 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge10.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge10.q.out @@ -542,7 +542,7 @@ PREHOOK: Input: default@orcfile_merge1@ds=1/part=0 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_101 +File Version: 0.12 with ORC_135 Rows: 242 Compression: SNAPPY Compression size: 4096 @@ -631,7 +631,7 @@ PREHOOK: Input: default@orcfile_merge1c@ds=1/part=0 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_101 +File Version: 0.12 with ORC_135 Rows: 242 Compression: SNAPPY Compression size: 4096 http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/ql/src/test/results/clientpositive/llap/orc_merge11.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/orc_merge11.q.out b/ql/src/test/results/clientpositive/llap/orc_merge11.q.out index 9c50f6d..dabf9ca 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge11.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge11.q.out @@ -72,7 +72,7 @@ PREHOOK: Input: default@orcfile_merge1 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_101 +File Version: 0.12 with ORC_135 Rows: 50000 Compression: ZLIB Compression size: 4096 @@ -96,22 +96,22 @@ File Statistics: Column 5: count: 50000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0 Stripes: - Stripe: offset: 3 data: 5897 rows: 50000 tail: 113 index: 498 + Stripe: offset: 3 data: 5897 rows: 50000 tail: 113 index: 497 Stream: column 0 section ROW_INDEX start: 3 length 17 Stream: column 1 section ROW_INDEX start: 20 length 83 Stream: column 2 section ROW_INDEX start: 103 length 81 Stream: column 3 section ROW_INDEX start: 184 length 111 Stream: column 4 section ROW_INDEX start: 295 length 110 - Stream: column 5 section ROW_INDEX start: 405 length 96 - Stream: column 1 section DATA start: 501 length 45 - Stream: column 2 section DATA start: 546 length 41 - Stream: column 2 section LENGTH start: 587 length 8 - Stream: column 2 section DICTIONARY_DATA start: 595 length 23 - Stream: column 3 section DATA start: 618 length 5167 - Stream: column 4 section DATA start: 5785 length 524 - Stream: column 4 section SECONDARY start: 6309 length 18 - Stream: column 5 section DATA start: 6327 length 53 - Stream: column 5 section SECONDARY start: 6380 length 18 + Stream: column 5 section ROW_INDEX start: 405 length 95 + Stream: column 1 section DATA start: 500 length 45 + Stream: column 2 section DATA start: 545 length 41 + Stream: column 2 section LENGTH start: 586 length 8 + Stream: column 2 section DICTIONARY_DATA start: 594 length 23 + Stream: column 3 section DATA start: 617 length 5167 + Stream: column 4 section DATA start: 5784 length 524 + Stream: column 4 section SECONDARY start: 6308 length 18 + Stream: column 5 section DATA start: 6326 length 53 + Stream: column 5 section SECONDARY start: 6379 length 18 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DICTIONARY_V2[6] @@ -155,7 +155,7 @@ Stripes: Entry 3: count: 10000 hasNull: false min: 1969-12-31 16:00:05.0 max: 1969-12-31 16:04:10.0 positions: 0,508,391,0,232,304 Entry 4: count: 10000 hasNull: false min: 1969-12-31 16:00:15.0 max: 1969-12-31 16:04:10.0 positions: 0,680,391,0,312,64 -File length: 6849 bytes +File length: 6854 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ @@ -163,7 +163,7 @@ ________________________________________________________________________________ -- END ORC FILE DUMP -- -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_101 +File Version: 0.12 with ORC_135 Rows: 50000 Compression: ZLIB Compression size: 4096 @@ -187,22 +187,22 @@ File Statistics: Column 5: count: 50000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0 Stripes: - Stripe: offset: 3 data: 5897 rows: 50000 tail: 113 index: 498 + Stripe: offset: 3 data: 5897 rows: 50000 tail: 113 index: 497 Stream: column 0 section ROW_INDEX start: 3 length 17 Stream: column 1 section ROW_INDEX start: 20 length 83 Stream: column 2 section ROW_INDEX start: 103 length 81 Stream: column 3 section ROW_INDEX start: 184 length 111 Stream: column 4 section ROW_INDEX start: 295 length 110 - Stream: column 5 section ROW_INDEX start: 405 length 96 - Stream: column 1 section DATA start: 501 length 45 - Stream: column 2 section DATA start: 546 length 41 - Stream: column 2 section LENGTH start: 587 length 8 - Stream: column 2 section DICTIONARY_DATA start: 595 length 23 - Stream: column 3 section DATA start: 618 length 5167 - Stream: column 4 section DATA start: 5785 length 524 - Stream: column 4 section SECONDARY start: 6309 length 18 - Stream: column 5 section DATA start: 6327 length 53 - Stream: column 5 section SECONDARY start: 6380 length 18 + Stream: column 5 section ROW_INDEX start: 405 length 95 + Stream: column 1 section DATA start: 500 length 45 + Stream: column 2 section DATA start: 545 length 41 + Stream: column 2 section LENGTH start: 586 length 8 + Stream: column 2 section DICTIONARY_DATA start: 594 length 23 + Stream: column 3 section DATA start: 617 length 5167 + Stream: column 4 section DATA start: 5784 length 524 + Stream: column 4 section SECONDARY start: 6308 length 18 + Stream: column 5 section DATA start: 6326 length 53 + Stream: column 5 section SECONDARY start: 6379 length 18 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DICTIONARY_V2[6] @@ -246,7 +246,7 @@ Stripes: Entry 3: count: 10000 hasNull: false min: 1969-12-31 16:00:05.0 max: 1969-12-31 16:04:10.0 positions: 0,508,391,0,232,304 Entry 4: count: 10000 hasNull: false min: 1969-12-31 16:00:15.0 max: 1969-12-31 16:04:10.0 positions: 0,680,391,0,312,64 -File length: 6849 bytes +File length: 6854 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ @@ -275,7 +275,7 @@ PREHOOK: Input: default@orcfile_merge1 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_101 +File Version: 0.12 with ORC_135 Rows: 100000 Compression: ZLIB Compression size: 4096 @@ -306,22 +306,22 @@ File Statistics: Column 5: count: 100000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0 Stripes: - Stripe: offset: 3 data: 5897 rows: 50000 tail: 113 index: 498 + Stripe: offset: 3 data: 5897 rows: 50000 tail: 113 index: 497 Stream: column 0 section ROW_INDEX start: 3 length 17 Stream: column 1 section ROW_INDEX start: 20 length 83 Stream: column 2 section ROW_INDEX start: 103 length 81 Stream: column 3 section ROW_INDEX start: 184 length 111 Stream: column 4 section ROW_INDEX start: 295 length 110 - Stream: column 5 section ROW_INDEX start: 405 length 96 - Stream: column 1 section DATA start: 501 length 45 - Stream: column 2 section DATA start: 546 length 41 - Stream: column 2 section LENGTH start: 587 length 8 - Stream: column 2 section DICTIONARY_DATA start: 595 length 23 - Stream: column 3 section DATA start: 618 length 5167 - Stream: column 4 section DATA start: 5785 length 524 - Stream: column 4 section SECONDARY start: 6309 length 18 - Stream: column 5 section DATA start: 6327 length 53 - Stream: column 5 section SECONDARY start: 6380 length 18 + Stream: column 5 section ROW_INDEX start: 405 length 95 + Stream: column 1 section DATA start: 500 length 45 + Stream: column 2 section DATA start: 545 length 41 + Stream: column 2 section LENGTH start: 586 length 8 + Stream: column 2 section DICTIONARY_DATA start: 594 length 23 + Stream: column 3 section DATA start: 617 length 5167 + Stream: column 4 section DATA start: 5784 length 524 + Stream: column 4 section SECONDARY start: 6308 length 18 + Stream: column 5 section DATA start: 6326 length 53 + Stream: column 5 section SECONDARY start: 6379 length 18 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DICTIONARY_V2[6] @@ -364,22 +364,22 @@ Stripes: Entry 2: count: 10000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0 positions: 0,336,391,0,156,32 Entry 3: count: 10000 hasNull: false min: 1969-12-31 16:00:05.0 max: 1969-12-31 16:04:10.0 positions: 0,508,391,0,232,304 Entry 4: count: 10000 hasNull: false min: 1969-12-31 16:00:15.0 max: 1969-12-31 16:04:10.0 positions: 0,680,391,0,312,64 - Stripe: offset: 6511 data: 5897 rows: 50000 tail: 113 index: 498 - Stream: column 0 section ROW_INDEX start: 6511 length 17 - Stream: column 1 section ROW_INDEX start: 6528 length 83 - Stream: column 2 section ROW_INDEX start: 6611 length 81 - Stream: column 3 section ROW_INDEX start: 6692 length 111 - Stream: column 4 section ROW_INDEX start: 6803 length 110 - Stream: column 5 section ROW_INDEX start: 6913 length 96 - Stream: column 1 section DATA start: 7009 length 45 - Stream: column 2 section DATA start: 7054 length 41 - Stream: column 2 section LENGTH start: 7095 length 8 - Stream: column 2 section DICTIONARY_DATA start: 7103 length 23 - Stream: column 3 section DATA start: 7126 length 5167 - Stream: column 4 section DATA start: 12293 length 524 - Stream: column 4 section SECONDARY start: 12817 length 18 - Stream: column 5 section DATA start: 12835 length 53 - Stream: column 5 section SECONDARY start: 12888 length 18 + Stripe: offset: 6510 data: 5897 rows: 50000 tail: 113 index: 497 + Stream: column 0 section ROW_INDEX start: 6510 length 17 + Stream: column 1 section ROW_INDEX start: 6527 length 83 + Stream: column 2 section ROW_INDEX start: 6610 length 81 + Stream: column 3 section ROW_INDEX start: 6691 length 111 + Stream: column 4 section ROW_INDEX start: 6802 length 110 + Stream: column 5 section ROW_INDEX start: 6912 length 95 + Stream: column 1 section DATA start: 7007 length 45 + Stream: column 2 section DATA start: 7052 length 41 + Stream: column 2 section LENGTH start: 7093 length 8 + Stream: column 2 section DICTIONARY_DATA start: 7101 length 23 + Stream: column 3 section DATA start: 7124 length 5167 + Stream: column 4 section DATA start: 12291 length 524 + Stream: column 4 section SECONDARY start: 12815 length 18 + Stream: column 5 section DATA start: 12833 length 53 + Stream: column 5 section SECONDARY start: 12886 length 18 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DICTIONARY_V2[6] @@ -423,7 +423,7 @@ Stripes: Entry 3: count: 10000 hasNull: false min: 1969-12-31 16:00:05.0 max: 1969-12-31 16:04:10.0 positions: 0,508,391,0,232,304 Entry 4: count: 10000 hasNull: false min: 1969-12-31 16:00:15.0 max: 1969-12-31 16:04:10.0 positions: 0,680,391,0,312,64 -File length: 13369 bytes +File length: 13372 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/ql/src/test/results/clientpositive/llap/orc_merge12.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/orc_merge12.q.out b/ql/src/test/results/clientpositive/llap/orc_merge12.q.out index 2c49e0f..a999b8a 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge12.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge12.q.out @@ -144,7 +144,7 @@ PREHOOK: Input: default@alltypesorc3xcols #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_101 +File Version: 0.12 with ORC_135 Rows: 24576 Compression: ZLIB Compression size: 262144 @@ -813,7 +813,7 @@ Stripes: Entry 0: count: 6889 hasNull: true true: 3402 positions: 0,0,0,0,0,0,0,0 Entry 1: count: 2284 hasNull: true true: 581 positions: 0,168,8,0,0,520,97,1 -File length: 3007981 bytes +File length: 3007982 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out b/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out index c2fb461..b13c0ee 100644 --- a/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out @@ -180,7 +180,7 @@ Table Parameters: numFiles 3 numRows 15 rawDataSize 3483 - totalSize 4003 + totalSize 4033 #### A masked pattern was here #### # Storage Information @@ -233,7 +233,7 @@ Table Parameters: numFiles 3 numRows 15 rawDataSize 3483 - totalSize 4003 + totalSize 4033 #### A masked pattern was here #### # Storage Information @@ -298,7 +298,7 @@ Table Parameters: numFiles 4 numRows 20 rawDataSize 4552 - totalSize 5360 + totalSize 5406 #### A masked pattern was here #### # Storage Information @@ -513,7 +513,7 @@ Table Parameters: numFiles 1 numRows 5 rawDataSize 1069 - totalSize 3224 + totalSize 3245 #### A masked pattern was here #### # Storage Information @@ -566,7 +566,7 @@ Table Parameters: numFiles 1 numRows 15 rawDataSize 3320 - totalSize 3224 + totalSize 3245 #### A masked pattern was here #### # Storage Information @@ -631,7 +631,7 @@ Table Parameters: numFiles 2 numRows 20 rawDataSize 4389 - totalSize 4581 + totalSize 4618 #### A masked pattern was here #### # Storage Information http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/ql/src/test/results/clientpositive/orc_file_dump.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/orc_file_dump.q.out b/ql/src/test/results/clientpositive/orc_file_dump.q.out index 67c75d0..5178bbf 100644 --- a/ql/src/test/results/clientpositive/orc_file_dump.q.out +++ b/ql/src/test/results/clientpositive/orc_file_dump.q.out @@ -93,7 +93,7 @@ PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_101 +File Version: 0.12 with ORC_135 Rows: 1049 Compression: ZLIB Compression size: 262144 @@ -129,7 +129,7 @@ File Statistics: Column 11: count: 1049 hasNull: false sum: 13278 Stripes: - Stripe: offset: 3 data: 22593 rows: 1049 tail: 250 index: 8963 + Stripe: offset: 3 data: 22593 rows: 1049 tail: 252 index: 8962 Stream: column 0 section ROW_INDEX start: 3 length 20 Stream: column 0 section BLOOM_FILTER_UTF8 start: 23 length 34 Stream: column 1 section ROW_INDEX start: 57 length 58 @@ -149,29 +149,29 @@ Stripes: Stream: column 8 section ROW_INDEX start: 6154 length 86 Stream: column 8 section BLOOM_FILTER_UTF8 start: 6240 length 1051 Stream: column 9 section ROW_INDEX start: 7291 length 50 - Stream: column 9 section BLOOM_FILTER_UTF8 start: 7341 length 53 - Stream: column 10 section ROW_INDEX start: 7394 length 82 - Stream: column 10 section BLOOM_FILTER_UTF8 start: 7476 length 1189 - Stream: column 11 section ROW_INDEX start: 8665 length 47 - Stream: column 11 section BLOOM_FILTER_UTF8 start: 8712 length 254 - Stream: column 1 section PRESENT start: 8966 length 17 - Stream: column 1 section DATA start: 8983 length 962 - Stream: column 2 section PRESENT start: 9945 length 17 - Stream: column 2 section DATA start: 9962 length 1441 - Stream: column 3 section DATA start: 11403 length 1704 - Stream: column 4 section DATA start: 13107 length 1998 - Stream: column 5 section DATA start: 15105 length 2925 - Stream: column 6 section DATA start: 18030 length 3323 - Stream: column 7 section DATA start: 21353 length 137 - Stream: column 8 section DATA start: 21490 length 1572 - Stream: column 8 section LENGTH start: 23062 length 310 - Stream: column 8 section DICTIONARY_DATA start: 23372 length 1548 - Stream: column 9 section DATA start: 24920 length 19 - Stream: column 9 section SECONDARY start: 24939 length 1783 - Stream: column 10 section DATA start: 26722 length 2138 - Stream: column 10 section SECONDARY start: 28860 length 231 - Stream: column 11 section DATA start: 29091 length 1877 - Stream: column 11 section LENGTH start: 30968 length 591 + Stream: column 9 section BLOOM_FILTER_UTF8 start: 7341 length 52 + Stream: column 10 section ROW_INDEX start: 7393 length 82 + Stream: column 10 section BLOOM_FILTER_UTF8 start: 7475 length 1189 + Stream: column 11 section ROW_INDEX start: 8664 length 47 + Stream: column 11 section BLOOM_FILTER_UTF8 start: 8711 length 254 + Stream: column 1 section PRESENT start: 8965 length 17 + Stream: column 1 section DATA start: 8982 length 962 + Stream: column 2 section PRESENT start: 9944 length 17 + Stream: column 2 section DATA start: 9961 length 1441 + Stream: column 3 section DATA start: 11402 length 1704 + Stream: column 4 section DATA start: 13106 length 1998 + Stream: column 5 section DATA start: 15104 length 2925 + Stream: column 6 section DATA start: 18029 length 3323 + Stream: column 7 section DATA start: 21352 length 137 + Stream: column 8 section DATA start: 21489 length 1572 + Stream: column 8 section LENGTH start: 23061 length 310 + Stream: column 8 section DICTIONARY_DATA start: 23371 length 1548 + Stream: column 9 section DATA start: 24919 length 19 + Stream: column 9 section SECONDARY start: 24938 length 1783 + Stream: column 10 section DATA start: 26721 length 2138 + Stream: column 10 section SECONDARY start: 28859 length 231 + Stream: column 11 section DATA start: 29090 length 1877 + Stream: column 11 section LENGTH start: 30967 length 591 Encoding column 0: DIRECT Encoding column 1: DIRECT Encoding column 2: DIRECT_V2 @@ -269,7 +269,7 @@ Stripes: Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 98 loadFactor: 0.0156 expectedFpp: 5.9604645E-8 Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 102 loadFactor: 0.0163 expectedFpp: 6.9948186E-8 -File length: 32435 bytes +File length: 32438 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ @@ -290,7 +290,7 @@ PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_101 +File Version: 0.12 with ORC_135 Rows: 1049 Compression: ZLIB Compression size: 262144 @@ -326,7 +326,7 @@ File Statistics: Column 11: count: 1049 hasNull: false sum: 13278 Stripes: - Stripe: offset: 3 data: 22593 rows: 1049 tail: 246 index: 13609 + Stripe: offset: 3 data: 22593 rows: 1049 tail: 249 index: 13609 Stream: column 0 section ROW_INDEX start: 3 length 20 Stream: column 0 section BLOOM_FILTER_UTF8 start: 23 length 43 Stream: column 1 section ROW_INDEX start: 66 length 58 @@ -466,7 +466,7 @@ Stripes: Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 174 loadFactor: 0.0181 expectedFpp: 6.426078E-13 Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 181 loadFactor: 0.0189 expectedFpp: 8.4693775E-13 -File length: 37078 bytes +File length: 37082 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ @@ -499,7 +499,7 @@ PREHOOK: Input: default@orc_ppd_part@ds=2015/hr=10 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_101 +File Version: 0.12 with ORC_135 Rows: 1049 Compression: ZLIB Compression size: 262144 @@ -535,7 +535,7 @@ File Statistics: Column 11: count: 1049 hasNull: false sum: 13278 Stripes: - Stripe: offset: 3 data: 22593 rows: 1049 tail: 250 index: 8963 + Stripe: offset: 3 data: 22593 rows: 1049 tail: 252 index: 8962 Stream: column 0 section ROW_INDEX start: 3 length 20 Stream: column 0 section BLOOM_FILTER_UTF8 start: 23 length 34 Stream: column 1 section ROW_INDEX start: 57 length 58 @@ -555,29 +555,29 @@ Stripes: Stream: column 8 section ROW_INDEX start: 6154 length 86 Stream: column 8 section BLOOM_FILTER_UTF8 start: 6240 length 1051 Stream: column 9 section ROW_INDEX start: 7291 length 50 - Stream: column 9 section BLOOM_FILTER_UTF8 start: 7341 length 53 - Stream: column 10 section ROW_INDEX start: 7394 length 82 - Stream: column 10 section BLOOM_FILTER_UTF8 start: 7476 length 1189 - Stream: column 11 section ROW_INDEX start: 8665 length 47 - Stream: column 11 section BLOOM_FILTER_UTF8 start: 8712 length 254 - Stream: column 1 section PRESENT start: 8966 length 17 - Stream: column 1 section DATA start: 8983 length 962 - Stream: column 2 section PRESENT start: 9945 length 17 - Stream: column 2 section DATA start: 9962 length 1441 - Stream: column 3 section DATA start: 11403 length 1704 - Stream: column 4 section DATA start: 13107 length 1998 - Stream: column 5 section DATA start: 15105 length 2925 - Stream: column 6 section DATA start: 18030 length 3323 - Stream: column 7 section DATA start: 21353 length 137 - Stream: column 8 section DATA start: 21490 length 1572 - Stream: column 8 section LENGTH start: 23062 length 310 - Stream: column 8 section DICTIONARY_DATA start: 23372 length 1548 - Stream: column 9 section DATA start: 24920 length 19 - Stream: column 9 section SECONDARY start: 24939 length 1783 - Stream: column 10 section DATA start: 26722 length 2138 - Stream: column 10 section SECONDARY start: 28860 length 231 - Stream: column 11 section DATA start: 29091 length 1877 - Stream: column 11 section LENGTH start: 30968 length 591 + Stream: column 9 section BLOOM_FILTER_UTF8 start: 7341 length 52 + Stream: column 10 section ROW_INDEX start: 7393 length 82 + Stream: column 10 section BLOOM_FILTER_UTF8 start: 7475 length 1189 + Stream: column 11 section ROW_INDEX start: 8664 length 47 + Stream: column 11 section BLOOM_FILTER_UTF8 start: 8711 length 254 + Stream: column 1 section PRESENT start: 8965 length 17 + Stream: column 1 section DATA start: 8982 length 962 + Stream: column 2 section PRESENT start: 9944 length 17 + Stream: column 2 section DATA start: 9961 length 1441 + Stream: column 3 section DATA start: 11402 length 1704 + Stream: column 4 section DATA start: 13106 length 1998 + Stream: column 5 section DATA start: 15104 length 2925 + Stream: column 6 section DATA start: 18029 length 3323 + Stream: column 7 section DATA start: 21352 length 137 + Stream: column 8 section DATA start: 21489 length 1572 + Stream: column 8 section LENGTH start: 23061 length 310 + Stream: column 8 section DICTIONARY_DATA start: 23371 length 1548 + Stream: column 9 section DATA start: 24919 length 19 + Stream: column 9 section SECONDARY start: 24938 length 1783 + Stream: column 10 section DATA start: 26721 length 2138 + Stream: column 10 section SECONDARY start: 28859 length 231 + Stream: column 11 section DATA start: 29090 length 1877 + Stream: column 11 section LENGTH start: 30967 length 591 Encoding column 0: DIRECT Encoding column 1: DIRECT Encoding column 2: DIRECT_V2 @@ -675,7 +675,7 @@ Stripes: Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 98 loadFactor: 0.0156 expectedFpp: 5.9604645E-8 Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 102 loadFactor: 0.0163 expectedFpp: 6.9948186E-8 -File length: 32435 bytes +File length: 32438 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/ql/src/test/results/clientpositive/orc_merge10.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/orc_merge10.q.out b/ql/src/test/results/clientpositive/orc_merge10.q.out index 61c6cdc..607aaeb 100644 --- a/ql/src/test/results/clientpositive/orc_merge10.q.out +++ b/ql/src/test/results/clientpositive/orc_merge10.q.out @@ -501,7 +501,7 @@ PREHOOK: Input: default@orcfile_merge1@ds=1/part=0 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_101 +File Version: 0.12 with ORC_135 Rows: 242 Compression: SNAPPY Compression size: 4096 @@ -571,7 +571,7 @@ PREHOOK: Input: default@orcfile_merge1c@ds=1/part=0 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_101 +File Version: 0.12 with ORC_135 Rows: 242 Compression: SNAPPY Compression size: 4096 http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/ql/src/test/results/clientpositive/orc_merge11.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/orc_merge11.q.out b/ql/src/test/results/clientpositive/orc_merge11.q.out index 9c50f6d..dabf9ca 100644 --- a/ql/src/test/results/clientpositive/orc_merge11.q.out +++ b/ql/src/test/results/clientpositive/orc_merge11.q.out @@ -72,7 +72,7 @@ PREHOOK: Input: default@orcfile_merge1 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_101 +File Version: 0.12 with ORC_135 Rows: 50000 Compression: ZLIB Compression size: 4096 @@ -96,22 +96,22 @@ File Statistics: Column 5: count: 50000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0 Stripes: - Stripe: offset: 3 data: 5897 rows: 50000 tail: 113 index: 498 + Stripe: offset: 3 data: 5897 rows: 50000 tail: 113 index: 497 Stream: column 0 section ROW_INDEX start: 3 length 17 Stream: column 1 section ROW_INDEX start: 20 length 83 Stream: column 2 section ROW_INDEX start: 103 length 81 Stream: column 3 section ROW_INDEX start: 184 length 111 Stream: column 4 section ROW_INDEX start: 295 length 110 - Stream: column 5 section ROW_INDEX start: 405 length 96 - Stream: column 1 section DATA start: 501 length 45 - Stream: column 2 section DATA start: 546 length 41 - Stream: column 2 section LENGTH start: 587 length 8 - Stream: column 2 section DICTIONARY_DATA start: 595 length 23 - Stream: column 3 section DATA start: 618 length 5167 - Stream: column 4 section DATA start: 5785 length 524 - Stream: column 4 section SECONDARY start: 6309 length 18 - Stream: column 5 section DATA start: 6327 length 53 - Stream: column 5 section SECONDARY start: 6380 length 18 + Stream: column 5 section ROW_INDEX start: 405 length 95 + Stream: column 1 section DATA start: 500 length 45 + Stream: column 2 section DATA start: 545 length 41 + Stream: column 2 section LENGTH start: 586 length 8 + Stream: column 2 section DICTIONARY_DATA start: 594 length 23 + Stream: column 3 section DATA start: 617 length 5167 + Stream: column 4 section DATA start: 5784 length 524 + Stream: column 4 section SECONDARY start: 6308 length 18 + Stream: column 5 section DATA start: 6326 length 53 + Stream: column 5 section SECONDARY start: 6379 length 18 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DICTIONARY_V2[6] @@ -155,7 +155,7 @@ Stripes: Entry 3: count: 10000 hasNull: false min: 1969-12-31 16:00:05.0 max: 1969-12-31 16:04:10.0 positions: 0,508,391,0,232,304 Entry 4: count: 10000 hasNull: false min: 1969-12-31 16:00:15.0 max: 1969-12-31 16:04:10.0 positions: 0,680,391,0,312,64 -File length: 6849 bytes +File length: 6854 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ @@ -163,7 +163,7 @@ ________________________________________________________________________________ -- END ORC FILE DUMP -- -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_101 +File Version: 0.12 with ORC_135 Rows: 50000 Compression: ZLIB Compression size: 4096 @@ -187,22 +187,22 @@ File Statistics: Column 5: count: 50000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0 Stripes: - Stripe: offset: 3 data: 5897 rows: 50000 tail: 113 index: 498 + Stripe: offset: 3 data: 5897 rows: 50000 tail: 113 index: 497 Stream: column 0 section ROW_INDEX start: 3 length 17 Stream: column 1 section ROW_INDEX start: 20 length 83 Stream: column 2 section ROW_INDEX start: 103 length 81 Stream: column 3 section ROW_INDEX start: 184 length 111 Stream: column 4 section ROW_INDEX start: 295 length 110 - Stream: column 5 section ROW_INDEX start: 405 length 96 - Stream: column 1 section DATA start: 501 length 45 - Stream: column 2 section DATA start: 546 length 41 - Stream: column 2 section LENGTH start: 587 length 8 - Stream: column 2 section DICTIONARY_DATA start: 595 length 23 - Stream: column 3 section DATA start: 618 length 5167 - Stream: column 4 section DATA start: 5785 length 524 - Stream: column 4 section SECONDARY start: 6309 length 18 - Stream: column 5 section DATA start: 6327 length 53 - Stream: column 5 section SECONDARY start: 6380 length 18 + Stream: column 5 section ROW_INDEX start: 405 length 95 + Stream: column 1 section DATA start: 500 length 45 + Stream: column 2 section DATA start: 545 length 41 + Stream: column 2 section LENGTH start: 586 length 8 + Stream: column 2 section DICTIONARY_DATA start: 594 length 23 + Stream: column 3 section DATA start: 617 length 5167 + Stream: column 4 section DATA start: 5784 length 524 + Stream: column 4 section SECONDARY start: 6308 length 18 + Stream: column 5 section DATA start: 6326 length 53 + Stream: column 5 section SECONDARY start: 6379 length 18 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DICTIONARY_V2[6] @@ -246,7 +246,7 @@ Stripes: Entry 3: count: 10000 hasNull: false min: 1969-12-31 16:00:05.0 max: 1969-12-31 16:04:10.0 positions: 0,508,391,0,232,304 Entry 4: count: 10000 hasNull: false min: 1969-12-31 16:00:15.0 max: 1969-12-31 16:04:10.0 positions: 0,680,391,0,312,64 -File length: 6849 bytes +File length: 6854 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ @@ -275,7 +275,7 @@ PREHOOK: Input: default@orcfile_merge1 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_101 +File Version: 0.12 with ORC_135 Rows: 100000 Compression: ZLIB Compression size: 4096 @@ -306,22 +306,22 @@ File Statistics: Column 5: count: 100000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0 Stripes: - Stripe: offset: 3 data: 5897 rows: 50000 tail: 113 index: 498 + Stripe: offset: 3 data: 5897 rows: 50000 tail: 113 index: 497 Stream: column 0 section ROW_INDEX start: 3 length 17 Stream: column 1 section ROW_INDEX start: 20 length 83 Stream: column 2 section ROW_INDEX start: 103 length 81 Stream: column 3 section ROW_INDEX start: 184 length 111 Stream: column 4 section ROW_INDEX start: 295 length 110 - Stream: column 5 section ROW_INDEX start: 405 length 96 - Stream: column 1 section DATA start: 501 length 45 - Stream: column 2 section DATA start: 546 length 41 - Stream: column 2 section LENGTH start: 587 length 8 - Stream: column 2 section DICTIONARY_DATA start: 595 length 23 - Stream: column 3 section DATA start: 618 length 5167 - Stream: column 4 section DATA start: 5785 length 524 - Stream: column 4 section SECONDARY start: 6309 length 18 - Stream: column 5 section DATA start: 6327 length 53 - Stream: column 5 section SECONDARY start: 6380 length 18 + Stream: column 5 section ROW_INDEX start: 405 length 95 + Stream: column 1 section DATA start: 500 length 45 + Stream: column 2 section DATA start: 545 length 41 + Stream: column 2 section LENGTH start: 586 length 8 + Stream: column 2 section DICTIONARY_DATA start: 594 length 23 + Stream: column 3 section DATA start: 617 length 5167 + Stream: column 4 section DATA start: 5784 length 524 + Stream: column 4 section SECONDARY start: 6308 length 18 + Stream: column 5 section DATA start: 6326 length 53 + Stream: column 5 section SECONDARY start: 6379 length 18 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DICTIONARY_V2[6] @@ -364,22 +364,22 @@ Stripes: Entry 2: count: 10000 hasNull: false min: 1969-12-31 16:00:00.0 max: 1969-12-31 16:04:10.0 positions: 0,336,391,0,156,32 Entry 3: count: 10000 hasNull: false min: 1969-12-31 16:00:05.0 max: 1969-12-31 16:04:10.0 positions: 0,508,391,0,232,304 Entry 4: count: 10000 hasNull: false min: 1969-12-31 16:00:15.0 max: 1969-12-31 16:04:10.0 positions: 0,680,391,0,312,64 - Stripe: offset: 6511 data: 5897 rows: 50000 tail: 113 index: 498 - Stream: column 0 section ROW_INDEX start: 6511 length 17 - Stream: column 1 section ROW_INDEX start: 6528 length 83 - Stream: column 2 section ROW_INDEX start: 6611 length 81 - Stream: column 3 section ROW_INDEX start: 6692 length 111 - Stream: column 4 section ROW_INDEX start: 6803 length 110 - Stream: column 5 section ROW_INDEX start: 6913 length 96 - Stream: column 1 section DATA start: 7009 length 45 - Stream: column 2 section DATA start: 7054 length 41 - Stream: column 2 section LENGTH start: 7095 length 8 - Stream: column 2 section DICTIONARY_DATA start: 7103 length 23 - Stream: column 3 section DATA start: 7126 length 5167 - Stream: column 4 section DATA start: 12293 length 524 - Stream: column 4 section SECONDARY start: 12817 length 18 - Stream: column 5 section DATA start: 12835 length 53 - Stream: column 5 section SECONDARY start: 12888 length 18 + Stripe: offset: 6510 data: 5897 rows: 50000 tail: 113 index: 497 + Stream: column 0 section ROW_INDEX start: 6510 length 17 + Stream: column 1 section ROW_INDEX start: 6527 length 83 + Stream: column 2 section ROW_INDEX start: 6610 length 81 + Stream: column 3 section ROW_INDEX start: 6691 length 111 + Stream: column 4 section ROW_INDEX start: 6802 length 110 + Stream: column 5 section ROW_INDEX start: 6912 length 95 + Stream: column 1 section DATA start: 7007 length 45 + Stream: column 2 section DATA start: 7052 length 41 + Stream: column 2 section LENGTH start: 7093 length 8 + Stream: column 2 section DICTIONARY_DATA start: 7101 length 23 + Stream: column 3 section DATA start: 7124 length 5167 + Stream: column 4 section DATA start: 12291 length 524 + Stream: column 4 section SECONDARY start: 12815 length 18 + Stream: column 5 section DATA start: 12833 length 53 + Stream: column 5 section SECONDARY start: 12886 length 18 Encoding column 0: DIRECT Encoding column 1: DIRECT_V2 Encoding column 2: DICTIONARY_V2[6] @@ -423,7 +423,7 @@ Stripes: Entry 3: count: 10000 hasNull: false min: 1969-12-31 16:00:05.0 max: 1969-12-31 16:04:10.0 positions: 0,508,391,0,232,304 Entry 4: count: 10000 hasNull: false min: 1969-12-31 16:00:15.0 max: 1969-12-31 16:04:10.0 positions: 0,680,391,0,312,64 -File length: 13369 bytes +File length: 13372 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/ql/src/test/results/clientpositive/orc_merge12.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/orc_merge12.q.out b/ql/src/test/results/clientpositive/orc_merge12.q.out index 2c49e0f..a999b8a 100644 --- a/ql/src/test/results/clientpositive/orc_merge12.q.out +++ b/ql/src/test/results/clientpositive/orc_merge12.q.out @@ -144,7 +144,7 @@ PREHOOK: Input: default@alltypesorc3xcols #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_101 +File Version: 0.12 with ORC_135 Rows: 24576 Compression: ZLIB Compression size: 262144 @@ -813,7 +813,7 @@ Stripes: Entry 0: count: 6889 hasNull: true true: 3402 positions: 0,0,0,0,0,0,0,0 Entry 1: count: 2284 hasNull: true true: 581 positions: 0,168,8,0,0,520,97,1 -File length: 3007981 bytes +File length: 3007982 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out b/ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out index 1ab0a39..da5b7d3 100644 --- a/ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_outer_join1.q.out @@ -244,7 +244,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cd - Statistics: Num rows: 15 Data size: 4003 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 4033 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] @@ -255,7 +255,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] - Statistics: Num rows: 15 Data size: 4003 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 4033 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator Spark Hash Table Sink Vectorization: className: VectorSparkHashTableSinkOperator @@ -288,7 +288,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c - Statistics: Num rows: 15 Data size: 4003 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 4033 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] @@ -299,7 +299,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] - Statistics: Num rows: 15 Data size: 4003 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 4033 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Outer Join0 to 1 @@ -319,13 +319,13 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23 input vertices: 1 Map 2 - Statistics: Num rows: 16 Data size: 4403 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 4436 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 16 Data size: 4403 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 4436 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -417,7 +417,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hd - Statistics: Num rows: 15 Data size: 4003 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 4033 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] @@ -428,7 +428,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0] - Statistics: Num rows: 15 Data size: 4003 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 4033 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator Spark Hash Table Sink Vectorization: className: VectorSparkHashTableSinkOperator @@ -461,7 +461,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c - Statistics: Num rows: 15 Data size: 4003 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 4033 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] @@ -472,7 +472,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0] - Statistics: Num rows: 15 Data size: 4003 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 4033 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Outer Join0 to 1 @@ -490,13 +490,13 @@ STAGE PLANS: outputColumnNames: _col0 input vertices: 1 Map 2 - Statistics: Num rows: 16 Data size: 4403 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 4436 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 16 Data size: 4403 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 4436 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -679,7 +679,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cd - Statistics: Num rows: 15 Data size: 4003 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 4033 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] @@ -690,7 +690,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [2] - Statistics: Num rows: 15 Data size: 4003 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 4033 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator Spark Hash Table Sink Vectorization: className: VectorSparkHashTableSinkOperator @@ -718,7 +718,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hd - Statistics: Num rows: 15 Data size: 4003 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 4033 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] @@ -729,7 +729,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0] - Statistics: Num rows: 15 Data size: 4003 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 4033 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator Spark Hash Table Sink Vectorization: className: VectorSparkHashTableSinkOperator @@ -764,7 +764,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c - Statistics: Num rows: 15 Data size: 4003 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 4033 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] @@ -775,7 +775,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0, 2] - Statistics: Num rows: 15 Data size: 4003 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 4033 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Outer Join0 to 1 @@ -793,7 +793,7 @@ STAGE PLANS: outputColumnNames: _col0 input vertices: 1 Map 3 - Statistics: Num rows: 16 Data size: 4403 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 16 Data size: 4436 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Outer Join0 to 1 @@ -811,7 +811,7 @@ STAGE PLANS: outputColumnNames: _col0 input vertices: 1 Map 4 - Statistics: Num rows: 17 Data size: 4843 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 17 Data size: 4879 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(), sum(_col0) Group By Vectorization: http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out b/ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out index 5a1a23d..1c9a9ea 100644 --- a/ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_outer_join2.q.out @@ -260,7 +260,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: cd - Statistics: Num rows: 20 Data size: 5237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 5277 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] @@ -271,7 +271,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [2] - Statistics: Num rows: 20 Data size: 5237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 5277 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator Spark Hash Table Sink Vectorization: className: VectorSparkHashTableSinkOperator @@ -299,7 +299,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hd - Statistics: Num rows: 20 Data size: 5237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 5277 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] @@ -310,7 +310,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [3] - Statistics: Num rows: 20 Data size: 5237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 5277 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator Spark Hash Table Sink Vectorization: className: VectorSparkHashTableSinkOperator @@ -345,7 +345,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: c - Statistics: Num rows: 20 Data size: 5237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 5277 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] @@ -356,7 +356,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [2, 3] - Statistics: Num rows: 20 Data size: 5237 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 20 Data size: 5277 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Outer Join0 to 1 @@ -374,7 +374,7 @@ STAGE PLANS: outputColumnNames: _col1 input vertices: 1 Map 3 - Statistics: Num rows: 22 Data size: 5760 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 22 Data size: 5804 Basic stats: COMPLETE Column stats: NONE Map Join Operator condition map: Left Outer Join0 to 1 @@ -392,7 +392,7 @@ STAGE PLANS: outputColumnNames: _col1 input vertices: 1 Map 4 - Statistics: Num rows: 24 Data size: 6336 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 24 Data size: 6384 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(), sum(_col1) Group By Vectorization: http://git-wip-us.apache.org/repos/asf/hive/blob/b4d1f96c/ql/src/test/results/clientpositive/tez/orc_merge12.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/tez/orc_merge12.q.out b/ql/src/test/results/clientpositive/tez/orc_merge12.q.out index 2c49e0f..a999b8a 100644 --- a/ql/src/test/results/clientpositive/tez/orc_merge12.q.out +++ b/ql/src/test/results/clientpositive/tez/orc_merge12.q.out @@ -144,7 +144,7 @@ PREHOOK: Input: default@alltypesorc3xcols #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_101 +File Version: 0.12 with ORC_135 Rows: 24576 Compression: ZLIB Compression size: 262144 @@ -813,7 +813,7 @@ Stripes: Entry 0: count: 6889 hasNull: true true: 3402 positions: 0,0,0,0,0,0,0,0 Entry 1: count: 2284 hasNull: true true: 581 positions: 0,168,8,0,0,520,97,1 -File length: 3007981 bytes +File length: 3007982 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________
