HIVE-10863: Merge master to Spark branch 7/29/2015 [Spark Branch] (reviewed by Chao)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/714b3db6 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/714b3db6 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/714b3db6 Branch: refs/heads/llap Commit: 714b3db65d41dd96db59ca1b9a6d1b6a4613072e Parents: 537114b 7df9d7a Author: xzhang <xzhang@xzdt> Authored: Thu Jul 30 17:41:17 2015 -0700 Committer: xzhang <xzhang@xzdt> Committed: Thu Jul 30 17:41:17 2015 -0700 ---------------------------------------------------------------------- .../java/org/apache/hive/beeline/BeeLine.java | 5 +- .../org/apache/hive/beeline/BeeLineOpts.java | 9 + .../apache/hive/beeline/DatabaseConnection.java | 9 + beeline/src/main/resources/BeeLine.properties | 2 + .../hadoop/hive/cli/TestOptionsProcessor.java | 1 - common/pom.xml | 5 + .../hadoop/hive/common/ValidReadTxnList.java | 2 +- .../hadoop/hive/common/type/HiveDecimal.java | 306 -- .../org/apache/hadoop/hive/conf/HiveConf.java | 3 + .../hive/hcatalog/mapreduce/PartInfo.java | 32 +- .../hive/hcatalog/mapreduce/SpecialCases.java | 8 +- .../mapreduce/TestHCatMultiOutputFormat.java | 6 +- .../hive/hcatalog/streaming/mutate/package.html | 31 +- .../mutate/worker/CreatePartitionHelper.java | 83 - .../mutate/worker/MetaStorePartitionHelper.java | 102 + .../mutate/worker/MutatorCoordinator.java | 21 +- .../worker/MutatorCoordinatorBuilder.java | 41 +- .../mutate/worker/PartitionHelper.java | 17 + .../mutate/worker/WarehousePartitionHelper.java | 69 + .../worker/TestMetaStorePartitionHelper.java | 112 + .../mutate/worker/TestMutatorCoordinator.java | 40 +- .../worker/TestWarehousePartitionHelper.java | 57 + .../antlr4/org/apache/hive/hplsql/Hplsql.g4 | 78 +- .../main/java/org/apache/hive/hplsql/Conn.java | 16 +- .../main/java/org/apache/hive/hplsql/Exec.java | 118 +- .../java/org/apache/hive/hplsql/Expression.java | 28 + .../main/java/org/apache/hive/hplsql/Query.java | 71 + .../main/java/org/apache/hive/hplsql/Stmt.java | 129 +- .../main/java/org/apache/hive/hplsql/Utils.java | 7 + .../main/java/org/apache/hive/hplsql/Var.java | 27 +- .../apache/hive/hplsql/functions/Function.java | 12 +- hplsql/src/main/resources/hplsql-site.xml | 95 + .../org/apache/hive/hplsql/TestHplsqlLocal.java | 26 +- .../db/create_procedure_return_cursor.sql | 53 + .../db/create_procedure_return_cursor2.sql | 59 + .../src/test/queries/db/cursor_attributes.sql | 60 + hplsql/src/test/queries/db/select_into.sql | 17 + .../src/test/queries/db/set_current_schema.sql | 6 + hplsql/src/test/queries/db/sys_refcursor.sql | 65 + hplsql/src/test/queries/db/use.sql | 2 + hplsql/src/test/queries/local/exception2.sql | 10 - hplsql/src/test/queries/local/exception3.sql | 5 - hplsql/src/test/queries/local/exception4.sql | 7 - hplsql/src/test/queries/local/exception5.sql | 10 - .../db/create_procedure_return_cursor.out.txt | 135 + .../db/create_procedure_return_cursor2.out.txt | 139 + .../test/results/db/cursor_attributes.out.txt | 33 + hplsql/src/test/results/db/select_into.out.txt | 19 + .../test/results/db/set_current_schema.out.txt | 12 + .../src/test/results/db/sys_refcursor.out.txt | 36 + hplsql/src/test/results/db/use.out.txt | 4 + .../test/resources/testconfiguration.properties | 5 + .../org/apache/hive/jdbc/HiveConnection.java | 4 + .../hadoop/hive/metastore/HiveMetaStore.java | 11 +- .../hadoop/hive/metastore/txn/TxnHandler.java | 9 +- .../metastore/txn/ValidCompactorTxnList.java | 2 +- pom.xml | 1 + ql/pom.xml | 1 + .../UDAFTemplates/VectorUDAFMinMaxString.txt | 3 +- .../java/org/apache/hadoop/hive/ql/Context.java | 1 - .../java/org/apache/hadoop/hive/ql/Driver.java | 202 +- .../org/apache/hadoop/hive/ql/ErrorMsg.java | 2 +- .../org/apache/hadoop/hive/ql/QueryPlan.java | 18 +- .../apache/hadoop/hive/ql/exec/ExplainTask.java | 7 +- .../apache/hadoop/hive/ql/exec/JoinUtil.java | 87 +- .../hadoop/hive/ql/exec/MapJoinOperator.java | 2 +- .../apache/hadoop/hive/ql/exec/MoveTask.java | 2 +- .../apache/hadoop/hive/ql/exec/Operator.java | 6 + .../apache/hadoop/hive/ql/exec/Utilities.java | 9 +- .../hive/ql/exec/tez/KeyValuesAdapter.java | 47 + .../hive/ql/exec/tez/KeyValuesFromKeyValue.java | 90 + .../ql/exec/tez/KeyValuesFromKeyValues.java | 48 + .../hive/ql/exec/tez/ReduceRecordProcessor.java | 11 +- .../hive/ql/exec/tez/ReduceRecordSource.java | 12 +- .../apache/hadoop/hive/ql/exec/tez/TezTask.java | 10 +- .../hive/ql/exec/vector/BytesColumnVector.java | 347 --- .../hive/ql/exec/vector/ColumnVector.java | 178 -- .../ql/exec/vector/DecimalColumnVector.java | 125 - .../hive/ql/exec/vector/DoubleColumnVector.java | 161 - .../hive/ql/exec/vector/LongColumnVector.java | 205 -- .../ql/exec/vector/VectorMapJoinOperator.java | 1 - .../hive/ql/exec/vector/VectorizedRowBatch.java | 186 -- .../mapjoin/VectorMapJoinCommonOperator.java | 1 + .../hadoop/hive/ql/io/orc/MemoryManager.java | 4 +- .../apache/hadoop/hive/ql/io/orc/OrcConf.java | 191 ++ .../apache/hadoop/hive/ql/io/orc/OrcFile.java | 158 +- .../hadoop/hive/ql/io/orc/OrcInputFormat.java | 4 +- .../hadoop/hive/ql/io/orc/OrcOutputFormat.java | 67 +- .../hive/ql/io/orc/OrcRawRecordMerger.java | 20 +- .../apache/hadoop/hive/ql/io/orc/Reader.java | 34 +- .../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 63 +- .../hive/ql/io/orc/TreeReaderFactory.java | 12 +- .../hadoop/hive/ql/io/orc/WriterImpl.java | 27 +- .../read/ParquetRecordReaderWrapper.java | 7 +- .../hive/ql/io/sarg/ConvertAstToSearchArg.java | 439 +++ .../hive/ql/io/sarg/SearchArgumentFactory.java | 56 - .../hive/ql/io/sarg/SearchArgumentImpl.java | 1027 ------- .../apache/hadoop/hive/ql/lib/RuleRegExp.java | 191 +- .../hadoop/hive/ql/lockmgr/DbTxnManager.java | 36 +- .../hadoop/hive/ql/lockmgr/DummyTxnManager.java | 8 + .../hadoop/hive/ql/lockmgr/HiveTxnManager.java | 21 + .../hive/ql/lockmgr/HiveTxnManagerImpl.java | 10 + .../hadoop/hive/ql/lockmgr/LockException.java | 8 +- .../hadoop/hive/ql/metadata/HiveException.java | 3 + .../hadoop/hive/ql/optimizer/ColumnPruner.java | 4 + .../hive/ql/optimizer/ColumnPrunerProcCtx.java | 95 +- .../ql/optimizer/ColumnPrunerProcFactory.java | 42 +- .../optimizer/ConstantPropagateProcFactory.java | 100 +- .../hive/ql/optimizer/ConvertJoinMapJoin.java | 213 +- .../hive/ql/optimizer/GenMapRedUtils.java | 18 +- .../hive/ql/optimizer/MapJoinProcessor.java | 44 +- .../ql/optimizer/ReduceSinkMapJoinProc.java | 84 +- .../hive/ql/optimizer/SimpleFetchOptimizer.java | 5 +- .../calcite/rules/HiveJoinToMultiJoinRule.java | 82 +- .../calcite/translator/HiveOpConverter.java | 2 +- .../calcite/translator/JoinTypeCheckCtx.java | 2 +- .../translator/PlanModifierForASTConv.java | 2 +- .../translator/PlanModifierForReturnPath.java | 6 +- .../stats/annotation/StatsRulesProcFactory.java | 3 + .../hive/ql/parse/BaseSemanticAnalyzer.java | 13 + .../hadoop/hive/ql/parse/CalcitePlanner.java | 3 +- .../hive/ql/parse/ExplainSemanticAnalyzer.java | 2 +- .../hadoop/hive/ql/parse/GenTezProcContext.java | 12 + .../hadoop/hive/ql/parse/GenTezUtils.java | 23 +- .../apache/hadoop/hive/ql/parse/GenTezWork.java | 81 +- .../org/apache/hadoop/hive/ql/parse/HiveLexer.g | 11 + .../apache/hadoop/hive/ql/parse/HiveParser.g | 70 + .../hadoop/hive/ql/parse/IdentifiersParser.g | 19 +- .../apache/hadoop/hive/ql/parse/ParseUtils.java | 51 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 61 +- .../hive/ql/parse/SemanticAnalyzerFactory.java | 12 + .../hadoop/hive/ql/parse/TypeCheckCtx.java | 15 +- .../hive/ql/parse/TypeCheckProcFactory.java | 4 + .../apache/hadoop/hive/ql/plan/BaseWork.java | 2 +- .../hive/ql/plan/CommonMergeJoinDesc.java | 4 + .../apache/hadoop/hive/ql/plan/ExplainWork.java | 19 +- .../hadoop/hive/ql/plan/ExprNodeDescUtils.java | 115 + .../hadoop/hive/ql/plan/HiveOperation.java | 32 +- .../apache/hadoop/hive/ql/plan/MapJoinDesc.java | 11 + .../hadoop/hive/ql/plan/PartitionDesc.java | 39 +- .../apache/hadoop/hive/ql/plan/ReduceWork.java | 2 +- .../ql/processors/CommandProcessorResponse.java | 21 +- .../hadoop/hive/ql/processors/HiveCommand.java | 3 + .../authorization/plugin/HiveOperationType.java | 5 + .../plugin/sqlstd/Operation2Privilege.java | 11 + .../hadoop/hive/ql/session/SessionState.java | 34 +- .../apache/hadoop/hive/ql/stats/StatsUtils.java | 10 +- .../hive/ql/udf/generic/GenericUDFBridge.java | 8 +- .../apache/hadoop/hive/ql/TestTxnCommands.java | 473 +++ .../apache/hadoop/hive/ql/TestTxnCommands2.java | 68 +- .../exec/vector/TestVectorizedRowBatchCtx.java | 6 +- .../hive/ql/io/orc/TestInputOutputFormat.java | 29 +- .../hadoop/hive/ql/io/orc/TestOrcFile.java | 11 +- .../hive/ql/io/orc/TestRecordReaderImpl.java | 63 +- .../hive/ql/io/orc/TestVectorizedORCReader.java | 75 +- .../parquet/TestParquetRecordReaderWrapper.java | 155 + .../ql/io/sarg/TestConvertAstToSearchArg.java | 2863 +++++++++++++++++ .../hive/ql/io/sarg/TestSearchArgumentImpl.java | 2891 +----------------- .../hadoop/hive/ql/lib/TestRuleRegExp.java | 118 + .../parse/TestUpdateDeleteSemanticAnalyzer.java | 34 +- .../positive/TestTransactionStatement.java | 102 + .../hive/ql/session/TestSessionState.java | 2 +- .../test/queries/clientpositive/stats_ppr_all.q | 24 + .../clientpositive/tez_dynpart_hashjoin_1.q | 101 + .../clientpositive/tez_dynpart_hashjoin_2.q | 83 + .../tez_vector_dynpart_hashjoin_1.q | 102 + .../tez_vector_dynpart_hashjoin_2.q | 84 + .../clientpositive/udf_from_utc_timestamp.q | 30 +- .../clientpositive/udf_to_utc_timestamp.q | 30 +- .../clientpositive/unionall_unbalancedppd.q | 120 + .../test/queries/clientpositive/vector_acid3.q | 17 + .../vector_aggregate_without_gby.q | 14 + .../clientnegative/exchange_partition.q.out | 2 +- .../clientpositive/exchange_partition.q.out | 4 +- .../clientpositive/exchange_partition2.q.out | 4 +- .../clientpositive/exchange_partition3.q.out | 4 +- .../results/clientpositive/spark/union16.q.out | 16 +- .../results/clientpositive/spark/union2.q.out | 16 +- .../results/clientpositive/spark/union9.q.out | 16 +- .../clientpositive/spark/union_view.q.out | 24 + .../results/clientpositive/stats_ppr_all.q.out | 284 ++ .../tez/tez_dynpart_hashjoin_1.q.out | 791 +++++ .../tez/tez_dynpart_hashjoin_2.q.out | 564 ++++ .../tez/tez_vector_dynpart_hashjoin_1.q.out | 804 +++++ .../tez/tez_vector_dynpart_hashjoin_2.q.out | 570 ++++ .../results/clientpositive/tez/union2.q.out | 28 +- .../results/clientpositive/tez/union9.q.out | 40 +- .../clientpositive/tez/vector_acid3.q.out | 31 + .../tez/vector_null_projection.q.out | 4 + .../clientpositive/udf_from_utc_timestamp.q.out | 66 +- .../clientpositive/udf_to_utc_timestamp.q.out | 66 +- .../test/results/clientpositive/union16.q.out | 354 ++- ql/src/test/results/clientpositive/union2.q.out | 32 +- ql/src/test/results/clientpositive/union9.q.out | 46 +- .../results/clientpositive/union_view.q.out | 24 + .../clientpositive/unionall_unbalancedppd.q.out | 373 +++ .../results/clientpositive/vector_acid3.q.out | 31 + .../vector_aggregate_without_gby.q.out | 96 + .../clientpositive/vector_null_projection.q.out | 4 + .../hadoop/hive/ql/io/sarg/ExpressionTree.java | 157 - .../hadoop/hive/ql/io/sarg/PredicateLeaf.java | 87 - .../hadoop/hive/ql/io/sarg/SearchArgument.java | 278 -- .../apache/hadoop/hive/serde2/SerDeUtils.java | 14 +- .../hadoop/hive/serde2/avro/InstanceCache.java | 9 +- .../hive/serde2/io/HiveDecimalWritable.java | 185 -- .../hive/serde2/lazy/LazyHiveDecimal.java | 3 +- .../lazy/fast/LazySimpleDeserializeRead.java | 16 +- .../lazybinary/LazyBinaryHiveDecimal.java | 2 +- .../hive/serde2/lazybinary/LazyBinarySerDe.java | 26 +- .../fast/LazyBinaryDeserializeRead.java | 6 +- .../fast/LazyBinarySerializeWrite.java | 5 +- .../hive/serde2/typeinfo/HiveDecimalUtils.java | 35 +- .../hive/serde2/avro/TestInstanceCache.java | 40 +- storage-api/pom.xml | 85 + .../hadoop/hive/common/type/HiveDecimal.java | 312 ++ .../hive/ql/exec/vector/BytesColumnVector.java | 322 ++ .../hive/ql/exec/vector/ColumnVector.java | 173 ++ .../ql/exec/vector/DecimalColumnVector.java | 106 + .../hive/ql/exec/vector/DoubleColumnVector.java | 143 + .../hive/ql/exec/vector/LongColumnVector.java | 189 ++ .../hive/ql/exec/vector/VectorizedRowBatch.java | 186 ++ .../hadoop/hive/ql/io/sarg/ExpressionTree.java | 156 + .../hadoop/hive/ql/io/sarg/PredicateLeaf.java | 104 + .../hadoop/hive/ql/io/sarg/SearchArgument.java | 287 ++ .../hive/ql/io/sarg/SearchArgumentFactory.java | 28 + .../hive/ql/io/sarg/SearchArgumentImpl.java | 687 +++++ .../hive/serde2/io/HiveDecimalWritable.java | 174 ++ 227 files changed, 16038 insertions(+), 7381 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/714b3db6/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/714b3db6/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/714b3db6/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/714b3db6/ql/pom.xml ---------------------------------------------------------------------- diff --cc ql/pom.xml index 9bd7bba,6026c49..d8451c5 --- a/ql/pom.xml +++ b/ql/pom.xml @@@ -721,6 -716,6 +721,7 @@@ <include>com.google.guava:guava</include> <include>net.sf.opencsv:opencsv</include> <include>org.apache.hive:spark-client</include> ++ <include>org.apache.hive:hive-storage-api</include> <include>joda-time:joda-time</include> </includes> </artifactSet> http://git-wip-us.apache.org/repos/asf/hive/blob/714b3db6/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/714b3db6/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/714b3db6/ql/src/test/results/clientpositive/spark/union16.q.out ---------------------------------------------------------------------- diff --cc ql/src/test/results/clientpositive/spark/union16.q.out index 39ba7b9,6e45714..5170bbd --- a/ql/src/test/results/clientpositive/spark/union16.q.out +++ b/ql/src/test/results/clientpositive/spark/union16.q.out @@@ -78,12 -78,494 +78,14 @@@ STAGE PLANS Map 1 Map Operator Tree: TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 10 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 11 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 12 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 13 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 14 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 15 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 16 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 17 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 18 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 19 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 20 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 21 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 22 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 23 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 24 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 25 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 26 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 3 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 4 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 5 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 6 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 7 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 8 - Map Operator Tree: - TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 9 - Map Operator Tree: - TableScan alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 12500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 12500 Data size: 132800 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) mode: hash http://git-wip-us.apache.org/repos/asf/hive/blob/714b3db6/ql/src/test/results/clientpositive/spark/union2.q.out ---------------------------------------------------------------------- diff --cc ql/src/test/results/clientpositive/spark/union2.q.out index 3c5b075,9e44bef..8b516b5 --- a/ql/src/test/results/clientpositive/spark/union2.q.out +++ b/ql/src/test/results/clientpositive/spark/union2.q.out @@@ -27,11 -27,33 +27,13 @@@ STAGE PLANS Map Operator Tree: TableScan alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1000 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 3 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) mode: hash http://git-wip-us.apache.org/repos/asf/hive/blob/714b3db6/ql/src/test/results/clientpositive/spark/union9.q.out ---------------------------------------------------------------------- diff --cc ql/src/test/results/clientpositive/spark/union9.q.out index 92499c0,cb41d91..44c5f6b --- a/ql/src/test/results/clientpositive/spark/union9.q.out +++ b/ql/src/test/results/clientpositive/spark/union9.q.out @@@ -29,19 -29,61 +29,21 @@@ STAGE PLANS Map Operator Tree: TableScan alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1500 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 3 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Map 4 - Map Operator Tree: - TableScan - alias: s1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: value (type: string) - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1500 Data size: 15936 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count(1) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reducer 2 Reduce Operator Tree: http://git-wip-us.apache.org/repos/asf/hive/blob/714b3db6/ql/src/test/results/clientpositive/spark/union_view.q.out ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/714b3db6/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java ---------------------------------------------------------------------- diff --cc serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java index 8c5b0b3,7588106..dc76c7d --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java @@@ -30,16 -30,15 +30,13 @@@ import org.apache.hadoop.hive.common.ty import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.serde2.fast.DeserializeRead; - import org.apache.hadoop.hive.serde2.fast.DeserializeRead.ReadIntervalDayTimeResults; - import org.apache.hadoop.hive.serde2.fast.DeserializeRead.ReadIntervalYearMonthResults; + import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.HiveCharWritable; --import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable; import org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable; import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; --import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; import org.apache.hadoop.hive.serde2.lazy.LazyBinary; import org.apache.hadoop.hive.serde2.lazy.LazyByte; import org.apache.hadoop.hive.serde2.lazy.LazyInteger; @@@ -47,14 -46,8 +44,8 @@@ import org.apache.hadoop.hive.serde2.la import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters; import org.apache.hadoop.hive.serde2.lazy.LazyShort; import org.apache.hadoop.hive.serde2.lazy.LazyUtils; - import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector; - import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters; - import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; - import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; import org.apache.hadoop.io.Text;
