HIVE-12015 : LLAP: merge master into branch (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c5ccf669 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c5ccf669 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c5ccf669 Branch: refs/heads/llap Commit: c5ccf6694490a63329b3f4d9040dd976abd9d790 Parents: a1bc2ef 5074423 Author: Sergey Shelukhin <[email protected]> Authored: Thu Oct 1 19:38:52 2015 -0700 Committer: Sergey Shelukhin <[email protected]> Committed: Thu Oct 1 19:38:52 2015 -0700 ---------------------------------------------------------------------- .../common/metrics/common/MetricsConstant.java | 5 + .../hcatalog/pig/TestHCatLoaderEncryption.java | 3 + .../hive/ql/security/FolderPermissionBase.java | 53 +- .../upgrade/derby/021-HIVE-11970.derby.sql | 6 + .../upgrade/derby/hive-schema-1.3.0.derby.sql | 12 +- .../upgrade/derby/hive-schema-2.0.0.derby.sql | 12 +- .../derby/upgrade-1.2.0-to-1.3.0.derby.sql | 1 + .../derby/upgrade-1.2.0-to-2.0.0.derby.sql | 3 +- .../upgrade/mssql/007-HIVE-11970.mssql.sql | 6 + .../upgrade/mssql/hive-schema-1.3.0.mssql.sql | 12 +- .../upgrade/mssql/hive-schema-2.0.0.mssql.sql | 12 +- .../mssql/upgrade-1.2.0-to-1.3.0.mssql.sql | 1 + .../mssql/upgrade-1.2.0-to-2.0.0.mssql.sql | 7 +- .../upgrade/mysql/022-HIVE-11970.mysql.sql | 6 + .../upgrade/mysql/hive-schema-1.3.0.mysql.sql | 12 +- .../upgrade/mysql/hive-schema-2.0.0.mysql.sql | 12 +- .../mysql/upgrade-1.2.0-to-1.3.0.mysql.sql | 1 + .../mysql/upgrade-1.2.0-to-2.0.0.mysql.sql | 2 + .../upgrade/oracle/022-HIVE-11970.oracle.sql | 23 + .../upgrade/oracle/hive-schema-1.3.0.oracle.sql | 12 +- .../upgrade/oracle/hive-schema-2.0.0.oracle.sql | 12 +- .../oracle/upgrade-1.2.0-to-1.3.0.oracle.sql | 2 + .../oracle/upgrade-1.2.0-to-2.0.0.oracle.sql | 2 + .../postgres/021-HIVE-11970.postgres.sql | 6 + .../postgres/hive-schema-1.3.0.postgres.sql | 12 +- .../postgres/hive-schema-2.0.0.postgres.sql | 12 +- .../upgrade-1.2.0-to-1.3.0.postgres.sql | 1 + .../upgrade-1.2.0-to-2.0.0.postgres.sql | 1 + .../hive/metastore/MetaStoreDirectSql.java | 34 +- .../hadoop/hive/metastore/txn/TxnHandler.java | 159 +++--- .../hadoop/hive/ql/exec/FileSinkOperator.java | 19 +- .../apache/hadoop/hive/ql/exec/Utilities.java | 16 + .../hadoop/hive/ql/io/HiveFileFormatUtils.java | 95 +++- .../hadoop/hive/ql/io/InputFormatChecker.java | 5 +- .../hadoop/hive/ql/io/RCFileInputFormat.java | 3 +- .../ql/io/SequenceFileInputFormatChecker.java | 3 +- .../hive/ql/io/VectorizedRCFileInputFormat.java | 3 +- .../hive/ql/io/orc/ColumnStatisticsImpl.java | 55 +- .../apache/hadoop/hive/ql/io/orc/InStream.java | 25 + .../hive/ql/io/orc/MetadataReaderImpl.java | 2 +- .../apache/hadoop/hive/ql/io/orc/OrcFile.java | 33 +- .../hadoop/hive/ql/io/orc/OrcInputFormat.java | 4 +- .../hadoop/hive/ql/io/orc/OrcOutputFormat.java | 145 ++++- .../apache/hadoop/hive/ql/io/orc/OrcUtils.java | 177 +----- .../hadoop/hive/ql/io/orc/ReaderImpl.java | 45 +- .../hadoop/hive/ql/io/orc/TypeDescription.java | 466 ++++++++++++++++ .../ql/io/orc/VectorizedOrcInputFormat.java | 2 +- .../apache/hadoop/hive/ql/io/orc/Writer.java | 9 + .../hadoop/hive/ql/io/orc/WriterImpl.java | 550 +++++++++---------- .../hadoop/hive/ql/lockmgr/DbLockManager.java | 21 + .../zookeeper/ZooKeeperHiveLockManager.java | 41 ++ .../apache/hadoop/hive/ql/metadata/Hive.java | 108 +--- .../hive/ql/optimizer/ColumnPrunerProcCtx.java | 2 +- .../optimizer/ConstantPropagateProcFactory.java | 11 +- .../hive/ql/optimizer/GenMapRedUtils.java | 10 +- .../calcite/translator/HiveGBOpConvUtil.java | 43 +- .../hadoop/hive/ql/parse/FromClauseParser.g | 30 +- .../apache/hadoop/hive/ql/parse/HiveParser.g | 7 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 33 +- .../hive/ql/plan/DynamicPartitionCtx.java | 27 - .../hadoop/hive/ql/io/orc/orc_proto.proto | 1 + .../hive/ql/exec/TestFileSinkOperator.java | 386 ++++++++----- .../hive/ql/io/orc/TestColumnStatistics.java | 43 +- .../hive/ql/io/orc/TestInputOutputFormat.java | 15 +- .../hadoop/hive/ql/io/orc/TestOrcFile.java | 41 +- .../hive/ql/io/orc/TestOrcRawRecordMerger.java | 2 +- .../hadoop/hive/ql/io/orc/TestOrcWideTable.java | 224 +------- .../hive/ql/io/orc/TestTypeDescription.java | 67 +++ .../zookeeper/TestZookeeperLockManager.java | 50 ++ .../hive/ql/txn/compactor/CompactorTest.java | 2 +- .../cbo_rp_gby2_map_multi_distinct.q | 38 ++ ql/src/test/queries/clientpositive/join_parse.q | 20 + .../queries/clientpositive/update_all_types.q | 2 +- .../clientpositive/windowing_windowspec2.q | 16 +- .../resources/orc-file-dump-bloomfilter.out | 2 +- .../resources/orc-file-dump-bloomfilter2.out | 2 +- .../orc-file-dump-dictionary-threshold.out | 2 +- ql/src/test/resources/orc-file-dump.json | 2 +- ql/src/test/resources/orc-file-dump.out | 2 +- ql/src/test/resources/orc-file-has-null.out | 2 +- .../clientnegative/cte_with_in_subquery.q.out | 2 +- .../clientpositive/annotate_stats_part.q.out | 6 +- .../clientpositive/annotate_stats_table.q.out | 4 +- .../cbo_rp_gby2_map_multi_distinct.q.out | 236 ++++++++ .../dynpart_sort_opt_vectorization.q.out | 16 +- .../dynpart_sort_optimization2.q.out | 8 +- .../extrapolate_part_stats_full.q.out | 24 +- .../extrapolate_part_stats_partial.q.out | 76 +-- .../extrapolate_part_stats_partial_ndv.q.out | 38 +- .../results/clientpositive/join_parse.q.out | 516 +++++++++++++++++ .../results/clientpositive/orc_analyze.q.out | 46 +- .../results/clientpositive/orc_file_dump.q.out | 18 +- .../clientpositive/orc_int_type_promotion.q.out | 6 +- .../clientpositive/spark/vectorized_ptf.q.out | 108 ++-- .../tez/dynpart_sort_opt_vectorization.q.out | 16 +- .../tez/dynpart_sort_optimization2.q.out | 8 +- .../clientpositive/tez/orc_analyze.q.out | 46 +- .../clientpositive/tez/union_fast_stats.q.out | 16 +- .../clientpositive/tez/update_all_types.q.out | 4 +- .../clientpositive/tez/vector_outer_join1.q.out | 48 +- .../clientpositive/tez/vector_outer_join4.q.out | 48 +- .../clientpositive/tez/vectorized_ptf.q.out | 108 ++-- .../clientpositive/union_fast_stats.q.out | 16 +- .../clientpositive/update_all_types.q.out | 4 +- .../results/clientpositive/vectorized_ptf.q.out | 104 ++-- .../clientpositive/windowing_windowspec2.q.out | 198 +++---- 106 files changed, 3256 insertions(+), 1815 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 1c350db,5b21af9..1d79aff --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@@ -3920,38 -3918,19 +3921,53 @@@ public final class Utilities } } + /** + * Returns the full path to the Jar containing the class. It always return a JAR. + * + * @param klass + * class. + * + * @return path to the Jar containing the class. + */ + @SuppressWarnings("rawtypes") + public static String jarFinderGetJar(Class klass) { + Preconditions.checkNotNull(klass, "klass"); + ClassLoader loader = klass.getClassLoader(); + if (loader != null) { + String class_file = klass.getName().replaceAll("\\.", "/") + ".class"; + try { + for (Enumeration itr = loader.getResources(class_file); itr.hasMoreElements();) { + URL url = (URL) itr.nextElement(); + String path = url.getPath(); + if (path.startsWith("file:")) { + path = path.substring("file:".length()); + } + path = URLDecoder.decode(path, "UTF-8"); + if ("jar".equals(url.getProtocol())) { + path = URLDecoder.decode(path, "UTF-8"); + return path.replaceAll("!.*$", ""); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return null; + } + + public static int getDPColOffset(FileSinkDesc conf) { + + if (conf.getWriteType() == AcidUtils.Operation.DELETE) { + // For deletes, there is only ROW__ID in non-partitioning, non-bucketing columns. + //See : UpdateDeleteSemanticAnalyzer::reparseAndSuperAnalyze() for details. + return 1; + } else if (conf.getWriteType() == AcidUtils.Operation.UPDATE) { + // For updates, ROW__ID is an extra column at index 0. + //See : UpdateDeleteSemanticAnalyzer::reparseAndSuperAnalyze() for details. + return getColumnNames(conf.getTableInfo().getProperties()).size() + 1; + } else { + return getColumnNames(conf.getTableInfo().getProperties()).size(); + } + + } } http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java index 8f0824b,6fec8b7..3dde0c4 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java @@@ -34,13 -35,12 +35,14 @@@ import com.google.protobuf.CodedInputSt public abstract class InStream extends InputStream { private static final Log LOG = LogFactory.getLog(InStream.class); + private static final int PROTOBUF_MESSAGE_MAX_LIMIT = 1024 << 20; // 1GB + protected final Long fileId; protected final String name; - protected final long length; + protected long length; - public InStream(String name, long length) { + public InStream(Long fileId, String name, long length) { + this.fileId = fileId; this.name = name; this.length = length; } @@@ -468,9 -444,31 +470,32 @@@ CompressionCodec codec, int bufferSize) throws IOException { if (codec == null) { - return new UncompressedStream(name, input, length); + return new UncompressedStream(fileId, name, input, length); } else { - return new CompressedStream(name, input, length, codec, bufferSize); + return new CompressedStream(fileId, name, input, length, codec, bufferSize); } } + + /** + * Creates coded input stream (used for protobuf message parsing) with higher message size limit. + * + * @param name the name of the stream + * @param input the list of ranges of bytes for the stream; from disk or cache + * @param length the length in bytes of the stream + * @param codec the compression codec + * @param bufferSize the compression buffer size + * @return coded input stream + * @throws IOException + */ - public static CodedInputStream createCodedInputStream(String name, ++ public static CodedInputStream createCodedInputStream(Long fileId, ++ String name, + List<DiskRange> input, + long length, + CompressionCodec codec, + int bufferSize) throws IOException { - InStream inStream = create(name, input, length, codec, bufferSize); ++ InStream inStream = create(fileId, name, input, length, codec, bufferSize); + CodedInputStream codedInputStream = CodedInputStream.newInstance(inStream); + codedInputStream.setSizeLimit(PROTOBUF_MESSAGE_MAX_LIMIT); + return codedInputStream; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java index 1456df3,0000000..5afba51 mode 100644,000000..100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java @@@ -1,123 -1,0 +1,123 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.io.DiskRange; +import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.BufferChunk; + +import com.google.common.collect.Lists; + +public class MetadataReaderImpl implements MetadataReader { + private final FSDataInputStream file; + private final CompressionCodec codec; + private final int bufferSize; + private final int typeCount; + + public MetadataReaderImpl(FileSystem fileSystem, Path path, + CompressionCodec codec, int bufferSize, int typeCount) throws IOException { + this(fileSystem.open(path), codec, bufferSize, typeCount); + } + + public MetadataReaderImpl(FSDataInputStream file, + CompressionCodec codec, int bufferSize, int typeCount) { + this.file = file; + this.codec = codec; + this.bufferSize = bufferSize; + this.typeCount = typeCount; + } + + @Override + public RecordReaderImpl.Index readRowIndex(StripeInformation stripe, + OrcProto.StripeFooter footer, boolean[] included, OrcProto.RowIndex[] indexes, + boolean[] sargColumns, OrcProto.BloomFilterIndex[] bloomFilterIndices) throws IOException { + if (footer == null) { + footer = readStripeFooter(stripe); + } + if (indexes == null) { + indexes = new OrcProto.RowIndex[typeCount]; + } + if (bloomFilterIndices == null) { + bloomFilterIndices = new OrcProto.BloomFilterIndex[typeCount]; + } + long offset = stripe.getOffset(); + List<OrcProto.Stream> streams = footer.getStreamsList(); + for (int i = 0; i < streams.size(); i++) { + OrcProto.Stream stream = streams.get(i); + OrcProto.Stream nextStream = null; + if (i < streams.size() - 1) { + nextStream = streams.get(i+1); + } + int col = stream.getColumn(); + int len = (int) stream.getLength(); + // row index stream and bloom filter are interlaced, check if the sarg column contains bloom + // filter and combine the io to read row index and bloom filters for that column together + if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.ROW_INDEX)) { + boolean readBloomFilter = false; + if (sargColumns != null && sargColumns[col] && + nextStream.getKind() == OrcProto.Stream.Kind.BLOOM_FILTER) { + len += nextStream.getLength(); + i += 1; + readBloomFilter = true; + } + if ((included == null || included[col]) && indexes[col] == null) { + byte[] buffer = new byte[len]; + file.readFully(offset, buffer, 0, buffer.length); + ByteBuffer bb = ByteBuffer.wrap(buffer); + indexes[col] = OrcProto.RowIndex.parseFrom(InStream.create(null, "index", + Lists.<DiskRange>newArrayList(new BufferChunk(bb, 0)), stream.getLength(), + codec, bufferSize)); + if (readBloomFilter) { + bb.position((int) stream.getLength()); + bloomFilterIndices[col] = OrcProto.BloomFilterIndex.parseFrom(InStream.create( + null, "bloom_filter", Lists.<DiskRange>newArrayList(new BufferChunk(bb, 0)), + nextStream.getLength(), codec, bufferSize)); + } + } + } + offset += len; + } + + RecordReaderImpl.Index index = new RecordReaderImpl.Index(indexes, bloomFilterIndices); + return index; + } + + @Override + public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException { + long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(); + int tailLength = (int) stripe.getFooterLength(); + + // read the footer + ByteBuffer tailBuf = ByteBuffer.allocate(tailLength); + file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength); - return OrcProto.StripeFooter.parseFrom(InStream.create(null, "footer", ++ return OrcProto.StripeFooter.parseFrom(InStream.createCodedInputStream(null, "footer", + Lists.<DiskRange>newArrayList(new BufferChunk(tailBuf, 0)), + tailLength, codec, bufferSize)); + } + + @Override + public void close() throws IOException { + file.close(); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java index 04654bc,57bde3e..8941db1 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java @@@ -107,8 -106,8 +107,8 @@@ import com.google.common.util.concurren * that added this event. Insert and update events include the entire row, while * delete events have null for row. */ - public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>, + public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>, - InputFormatChecker, VectorizedInputFormatInterface, + InputFormatChecker, VectorizedInputFormatInterface, LlapWrappableInputFormatInterface, AcidInputFormat<NullWritable, OrcStruct>, CombineHiveInputFormat.AvoidSplitCombination { static enum SplitStrategyKind{ http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java index ce630bb,3bac48a..f3689fe --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java @@@ -423,47 -383,16 +419,16 @@@ public class ReaderImpl implements Read int footerSize, CompressionCodec codec, int bufferSize) throws IOException { bb.position(footerAbsPos); bb.limit(footerAbsPos + footerSize); - InputStream instream = InStream.create(null, "footer", Lists.<DiskRange>newArrayList( - new BufferChunk(bb, 0)), footerSize, codec, bufferSize); - return OrcProto.Footer.parseFrom(instream); - return OrcProto.Footer.parseFrom(InStream.createCodedInputStream("footer", ++ return OrcProto.Footer.parseFrom(InStream.createCodedInputStream(null, "footer", + Lists.<DiskRange>newArrayList(new BufferChunk(bb, 0)), footerSize, codec, bufferSize)); } private static OrcProto.Metadata extractMetadata(ByteBuffer bb, int metadataAbsPos, int metadataSize, CompressionCodec codec, int bufferSize) throws IOException { bb.position(metadataAbsPos); bb.limit(metadataAbsPos + metadataSize); - InputStream instream = InStream.create(null, "metadata", Lists.<DiskRange>newArrayList( - new BufferChunk(bb, 0)), metadataSize, codec, bufferSize); - CodedInputStream in = CodedInputStream.newInstance(instream); - int msgLimit = DEFAULT_PROTOBUF_MESSAGE_LIMIT; - OrcProto.Metadata meta = null; - do { - try { - in.setSizeLimit(msgLimit); - meta = OrcProto.Metadata.parseFrom(in); - } catch (InvalidProtocolBufferException e) { - if (e.getMessage().contains("Protocol message was too large")) { - LOG.warn("Metadata section is larger than " + msgLimit + " bytes. Increasing the max" + - " size of the coded input stream." ); - - msgLimit = msgLimit << 1; - if (msgLimit > PROTOBUF_MESSAGE_MAX_LIMIT) { - LOG.error("Metadata section exceeds max protobuf message size of " + - PROTOBUF_MESSAGE_MAX_LIMIT + " bytes."); - throw e; - } - - // we must have failed in the middle of reading instream and instream doesn't support - // resetting the stream - instream = InStream.create(null, "metadata", Lists.<DiskRange>newArrayList( - new BufferChunk(bb, 0)), metadataSize, codec, bufferSize); - in = CodedInputStream.newInstance(instream); - } else { - throw e; - } - } - } while (meta == null); - return meta; - return OrcProto.Metadata.parseFrom(InStream.createCodedInputStream("metadata", ++ return OrcProto.Metadata.parseFrom(InStream.createCodedInputStream(null, "metadata", + Lists.<DiskRange>newArrayList(new BufferChunk(bb, 0)), metadataSize, codec, bufferSize)); } private static OrcProto.PostScript extractPostScript(ByteBuffer bb, Path path, http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestColumnStatistics.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java ---------------------------------------------------------------------- diff --cc ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java index 7e81615,06e3362..e78f7aa --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java @@@ -656,17 -657,18 +657,18 @@@ public class TestOrcFile Reader reader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs)); - assertEquals(3, OrcUtils.getFlattenedColumnsCount(inspector)); + TypeDescription schema = writer.getSchema(); + assertEquals(2, schema.getMaximumId()); boolean[] expected = new boolean[] {false, true, false}; - boolean[] included = OrcUtils.includeColumns("int1", "int1,string1", inspector); + boolean[] included = OrcUtils.includeColumns("int1", schema); assertEquals(true, Arrays.equals(expected, included)); - Metadata metadata = reader.getMetadata(); - int numStripes = metadata.getStripeStatistics().size(); + List<StripeStatistics> stats = reader.getStripeStatistics(); + int numStripes = stats.size(); assertEquals(3, numStripes); - StripeStatistics ss1 = metadata.getStripeStatistics().get(0); - StripeStatistics ss2 = metadata.getStripeStatistics().get(1); - StripeStatistics ss3 = metadata.getStripeStatistics().get(2); + StripeStatistics ss1 = stats.get(0); + StripeStatistics ss2 = stats.get(1); + StripeStatistics ss3 = stats.get(2); assertEquals(5000, ss1.getColumnStatistics()[0].getNumberOfValues()); assertEquals(5000, ss2.getColumnStatistics()[0].getNumberOfValues()); @@@ -777,9 -777,11 +777,9 @@@ true, true, true, true}; included = OrcUtils.includeColumns( "boolean1,byte1,short1,int1,long1,float1,double1,bytes1,string1,middle,list,map", - "boolean1,byte1,short1,int1,long1,float1,double1,bytes1,string1,middle,list,map", inspector); + schema); assertEquals(true, Arrays.equals(expected, included)); - Metadata metadata = reader.getMetadata(); - // check the stats ColumnStatistics[] stats = reader.getStatistics(); assertEquals(2, stats[1].getNumberOfValues()); http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out ---------------------------------------------------------------------- diff --cc ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out index 46a25e7,92e7163..c94c3f2 --- a/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_outer_join1.q.out @@@ -562,10 -562,9 +562,10 @@@ STAGE PLANS key expressions: _col0 (type: tinyint) sort order: + Map-reduce partition columns: _col0 (type: tinyint) - Statistics: Num rows: 15 Data size: 3483 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 15 Data size: 3651 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Reducer 2 + Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0), sum(VALUE._col1) http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out ---------------------------------------------------------------------- diff --cc ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out index f0a9185,aa201ad..f66d5a4 --- a/ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_outer_join4.q.out @@@ -931,10 -931,9 +931,10 @@@ STAGE PLANS key expressions: _col0 (type: tinyint) sort order: + Map-reduce partition columns: _col0 (type: tinyint) - Statistics: Num rows: 30 Data size: 6838 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 30 Data size: 7006 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Reducer 2 + Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c5ccf669/ql/src/test/results/clientpositive/vectorized_ptf.q.out ----------------------------------------------------------------------
