Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/hive
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/451381cf Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/451381cf Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/451381cf Branch: refs/heads/beeline-cli Commit: 451381cfb28d1b9ff2775d55d92924956b3a5cdb Parents: 0679078 5238303 Author: Dmitry Tolpeko <[email protected]> Authored: Tue Sep 22 06:41:46 2015 -0700 Committer: Dmitry Tolpeko <[email protected]> Committed: Tue Sep 22 06:41:46 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ant/QTestGenTask.java | 11 + beeline/src/main/resources/beeline-log4j2.xml | 5 +- bin/beeline | 5 + bin/ext/hbaseimport.cmd | 35 + bin/ext/hbaseimport.sh | 27 + bin/ext/hbaseschematool.sh | 27 + .../apache/hadoop/hive/common/ObjectPair.java | 5 + .../hadoop/hive/common/jsonexplain/tez/Op.java | 8 +- .../hive/common/jsonexplain/tez/Stage.java | 14 +- .../common/jsonexplain/tez/TezJsonParser.java | 17 +- .../org/apache/hadoop/hive/conf/HiveConf.java | 111 +- .../apache/hadoop/hive/ql/log/PerfLogger.java | 196 + .../apache/hive/common/util/BloomFilter.java | 20 +- common/src/main/resources/hive-log4j2.xml | 5 +- .../test/resources/hive-exec-log4j2-test.xml | 5 +- common/src/test/resources/hive-log4j2-test.xml | 5 +- data/conf/hive-log4j2.xml | 5 +- data/conf/tez/hive-site.xml | 10 + data/files/dynpartdata1.txt | 5 + data/files/dynpartdata2.txt | 6 + .../HiveHBaseTableSnapshotInputFormat.java | 21 +- .../queries/positive/hbase_handler_snapshot.q | 4 + .../positive/hbase_handler_snapshot.q.out | 22 + .../deployers/config/hive/hive-log4j2.xml | 5 +- .../svr/src/main/config/webhcat-log4j2.xml | 5 +- .../benchmark/serde/LazySimpleSerDeBench.java | 453 + .../hive/thrift/TestHadoop20SAuthBridge.java | 420 - .../hive/thrift/TestHadoopAuthBridge23.java | 423 + itests/hive-unit/pom.xml | 35 + .../hadoop/hive/metastore/TestAdminUser.java | 4 +- .../hive/metastore/TestHiveMetaStore.java | 3 + .../metastore/hbase/HBaseIntegrationTests.java | 117 + .../TestHBaseAggrStatsCacheIntegration.java | 691 + .../hive/metastore/hbase/TestHBaseImport.java | 650 + .../metastore/hbase/TestHBaseMetastoreSql.java | 223 + .../hbase/TestHBaseStoreIntegration.java | 1794 + .../hbase/TestStorageDescriptorSharing.java | 191 + .../hive/ql/security/FolderPermissionBase.java | 17 +- itests/qtest/pom.xml | 10 +- .../test/resources/testconfiguration.properties | 3 + itests/util/pom.xml | 32 + .../metastore/hbase/HBaseStoreTestUtil.java | 45 + .../org/apache/hadoop/hive/ql/QTestUtil.java | 50 +- .../hive/jdbc/ZooKeeperHiveClientHelper.java | 32 +- metastore/if/hive_metastore.thrift | 54 + metastore/pom.xml | 82 + .../metastore/hbase/HbaseMetastoreProto.java | 34901 +++++++++++++++++ .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 6919 ++-- .../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 664 + .../ThriftHiveMetastore_server.skeleton.cpp | 25 + .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 1294 +- .../gen/thrift/gen-cpp/hive_metastore_types.h | 371 + .../hive/metastore/api/AbortTxnRequest.java | 2 +- .../metastore/api/AddDynamicPartitions.java | 2 +- .../metastore/api/AddPartitionsRequest.java | 2 +- .../hive/metastore/api/AddPartitionsResult.java | 2 +- .../hadoop/hive/metastore/api/AggrStats.java | 2 +- .../metastore/api/AlreadyExistsException.java | 2 +- .../metastore/api/BinaryColumnStatsData.java | 2 +- .../metastore/api/BooleanColumnStatsData.java | 2 +- .../hive/metastore/api/CheckLockRequest.java | 2 +- .../metastore/api/ClearFileMetadataRequest.java | 438 + .../metastore/api/ClearFileMetadataResult.java | 283 + .../hive/metastore/api/ColumnStatistics.java | 2 +- .../metastore/api/ColumnStatisticsDesc.java | 2 +- .../hive/metastore/api/ColumnStatisticsObj.java | 2 +- .../hive/metastore/api/CommitTxnRequest.java | 2 +- .../hive/metastore/api/CompactionRequest.java | 2 +- .../api/ConfigValSecurityException.java | 2 +- .../api/CurrentNotificationEventId.java | 2 +- .../hadoop/hive/metastore/api/Database.java | 2 +- .../apache/hadoop/hive/metastore/api/Date.java | 2 +- .../hive/metastore/api/DateColumnStatsData.java | 2 +- .../hadoop/hive/metastore/api/Decimal.java | 2 +- .../metastore/api/DecimalColumnStatsData.java | 2 +- .../metastore/api/DoubleColumnStatsData.java | 2 +- .../hive/metastore/api/DropPartitionsExpr.java | 2 +- .../metastore/api/DropPartitionsRequest.java | 2 +- .../metastore/api/DropPartitionsResult.java | 2 +- .../hive/metastore/api/EnvironmentContext.java | 2 +- .../hadoop/hive/metastore/api/FieldSchema.java | 2 +- .../hive/metastore/api/FireEventRequest.java | 2 +- .../hive/metastore/api/FireEventResponse.java | 2 +- .../hadoop/hive/metastore/api/Function.java | 2 +- .../metastore/api/GetAllFunctionsResponse.java | 38 +- .../api/GetFileMetadataByExprRequest.java | 548 + .../api/GetFileMetadataByExprResult.java | 703 + .../metastore/api/GetFileMetadataRequest.java | 438 + .../metastore/api/GetFileMetadataResult.java | 540 + .../metastore/api/GetOpenTxnsInfoResponse.java | 2 +- .../hive/metastore/api/GetOpenTxnsResponse.java | 2 +- .../api/GetPrincipalsInRoleRequest.java | 2 +- .../api/GetPrincipalsInRoleResponse.java | 2 +- .../api/GetRoleGrantsForPrincipalRequest.java | 2 +- .../api/GetRoleGrantsForPrincipalResponse.java | 2 +- .../api/GrantRevokePrivilegeRequest.java | 2 +- .../api/GrantRevokePrivilegeResponse.java | 2 +- .../metastore/api/GrantRevokeRoleRequest.java | 2 +- .../metastore/api/GrantRevokeRoleResponse.java | 2 +- .../hive/metastore/api/HeartbeatRequest.java | 2 +- .../metastore/api/HeartbeatTxnRangeRequest.java | 2 +- .../api/HeartbeatTxnRangeResponse.java | 2 +- .../hive/metastore/api/HiveObjectPrivilege.java | 2 +- .../hive/metastore/api/HiveObjectRef.java | 2 +- .../apache/hadoop/hive/metastore/api/Index.java | 2 +- .../api/IndexAlreadyExistsException.java | 2 +- .../metastore/api/InsertEventRequestData.java | 2 +- .../metastore/api/InvalidInputException.java | 2 +- .../metastore/api/InvalidObjectException.java | 2 +- .../api/InvalidOperationException.java | 2 +- .../api/InvalidPartitionException.java | 2 +- .../hive/metastore/api/LockComponent.java | 2 +- .../hadoop/hive/metastore/api/LockRequest.java | 2 +- .../hadoop/hive/metastore/api/LockResponse.java | 2 +- .../hive/metastore/api/LongColumnStatsData.java | 2 +- .../hive/metastore/api/MetaException.java | 2 +- .../hive/metastore/api/MetadataPpdResult.java | 508 + .../hive/metastore/api/NoSuchLockException.java | 2 +- .../metastore/api/NoSuchObjectException.java | 2 +- .../hive/metastore/api/NoSuchTxnException.java | 2 +- .../hive/metastore/api/NotificationEvent.java | 2 +- .../metastore/api/NotificationEventRequest.java | 2 +- .../api/NotificationEventResponse.java | 2 +- .../hive/metastore/api/OpenTxnRequest.java | 2 +- .../hive/metastore/api/OpenTxnsResponse.java | 2 +- .../apache/hadoop/hive/metastore/api/Order.java | 2 +- .../hadoop/hive/metastore/api/Partition.java | 2 +- .../api/PartitionListComposingSpec.java | 2 +- .../hive/metastore/api/PartitionSpec.java | 2 +- .../api/PartitionSpecWithSharedSD.java | 2 +- .../hive/metastore/api/PartitionWithoutSD.java | 2 +- .../metastore/api/PartitionsByExprRequest.java | 2 +- .../metastore/api/PartitionsByExprResult.java | 2 +- .../metastore/api/PartitionsStatsRequest.java | 2 +- .../metastore/api/PartitionsStatsResult.java | 2 +- .../metastore/api/PrincipalPrivilegeSet.java | 2 +- .../hadoop/hive/metastore/api/PrivilegeBag.java | 2 +- .../hive/metastore/api/PrivilegeGrantInfo.java | 2 +- .../metastore/api/PutFileMetadataRequest.java | 588 + .../metastore/api/PutFileMetadataResult.java | 283 + .../hadoop/hive/metastore/api/ResourceUri.java | 2 +- .../apache/hadoop/hive/metastore/api/Role.java | 2 +- .../hive/metastore/api/RolePrincipalGrant.java | 2 +- .../hadoop/hive/metastore/api/Schema.java | 2 +- .../hadoop/hive/metastore/api/SerDeInfo.java | 2 +- .../api/SetPartitionsStatsRequest.java | 2 +- .../hive/metastore/api/ShowCompactRequest.java | 2 +- .../hive/metastore/api/ShowCompactResponse.java | 2 +- .../api/ShowCompactResponseElement.java | 2 +- .../hive/metastore/api/ShowLocksRequest.java | 2 +- .../hive/metastore/api/ShowLocksResponse.java | 2 +- .../metastore/api/ShowLocksResponseElement.java | 2 +- .../hadoop/hive/metastore/api/SkewedInfo.java | 2 +- .../hive/metastore/api/StorageDescriptor.java | 2 +- .../metastore/api/StringColumnStatsData.java | 2 +- .../apache/hadoop/hive/metastore/api/Table.java | 2 +- .../hive/metastore/api/TableStatsRequest.java | 2 +- .../hive/metastore/api/TableStatsResult.java | 2 +- .../hive/metastore/api/ThriftHiveMetastore.java | 8422 ++-- .../hive/metastore/api/TxnAbortedException.java | 2 +- .../hadoop/hive/metastore/api/TxnInfo.java | 2 +- .../hive/metastore/api/TxnOpenException.java | 2 +- .../apache/hadoop/hive/metastore/api/Type.java | 2 +- .../hive/metastore/api/UnknownDBException.java | 2 +- .../api/UnknownPartitionException.java | 2 +- .../metastore/api/UnknownTableException.java | 2 +- .../hive/metastore/api/UnlockRequest.java | 2 +- .../hadoop/hive/metastore/api/Version.java | 2 +- .../gen-php/metastore/ThriftHiveMetastore.php | 2810 +- .../src/gen/thrift/gen-php/metastore/Types.php | 1009 +- .../hive_metastore/ThriftHiveMetastore-remote | 49 +- .../hive_metastore/ThriftHiveMetastore.py | 1563 +- .../gen/thrift/gen-py/hive_metastore/ttypes.py | 734 +- .../gen/thrift/gen-rb/hive_metastore_types.rb | 167 + .../gen/thrift/gen-rb/thrift_hive_metastore.rb | 267 + .../hadoop/hive/metastore/HiveAlterHandler.java | 38 +- .../hadoop/hive/metastore/HiveMetaStore.java | 272 +- .../hive/metastore/HiveMetaStoreClient.java | 112 +- .../hadoop/hive/metastore/IMetaStoreClient.java | 6 + .../hadoop/hive/metastore/ObjectStore.java | 437 +- .../hive/metastore/PartFilterExprUtil.java | 149 + .../apache/hadoop/hive/metastore/RawStore.java | 66 +- .../hadoop/hive/metastore/RawStoreProxy.java | 5 +- .../hive/metastore/RetryingHMSHandler.java | 33 +- .../hbase/AggrStatsInvalidatorFilter.java | 121 + .../hadoop/hive/metastore/hbase/Counter.java | 53 + .../hive/metastore/hbase/HBaseConnection.java | 96 + .../metastore/hbase/HBaseFilterPlanUtil.java | 612 + .../hive/metastore/hbase/HBaseImport.java | 535 + .../hive/metastore/hbase/HBaseReadWrite.java | 2106 + .../hive/metastore/hbase/HBaseSchemaTool.java | 239 + .../hadoop/hive/metastore/hbase/HBaseStore.java | 2387 ++ .../hadoop/hive/metastore/hbase/HBaseUtils.java | 1340 + .../hive/metastore/hbase/ObjectCache.java | 81 + .../hive/metastore/hbase/PartitionCache.java | 168 + .../metastore/hbase/PartitionKeyComparator.java | 292 + .../hbase/SharedStorageDescriptor.java | 251 + .../hadoop/hive/metastore/hbase/StatsCache.java | 326 + .../metastore/hbase/TephraHBaseConnection.java | 127 + .../metastore/hbase/VanillaHBaseConnection.java | 137 + .../stats/BinaryColumnStatsAggregator.java | 35 + .../stats/BooleanColumnStatsAggregator.java | 35 + .../hbase/stats/ColumnStatsAggregator.java | 26 + .../stats/ColumnStatsAggregatorFactory.java | 94 + .../stats/DecimalColumnStatsAggregator.java | 43 + .../stats/DoubleColumnStatsAggregator.java | 36 + .../hbase/stats/LongColumnStatsAggregator.java | 36 + .../stats/StringColumnStatsAggregator.java | 36 + .../hive/metastore/parser/ExpressionTree.java | 9 +- .../hive/metastore/tools/HiveMetaTool.java | 5 + .../hadoop/hive/metastore/txn/TxnHandler.java | 61 +- .../metastore/hbase/hbase_metastore_proto.proto | 282 + .../DummyRawStoreControlledCommit.java | 56 +- .../DummyRawStoreForJdoConnection.java | 50 +- .../hadoop/hive/metastore/TestObjectStore.java | 43 +- .../hadoop/hive/metastore/hbase/MockUtils.java | 199 + .../hbase/TestHBaseAggregateStatsCache.java | 316 + .../hbase/TestHBaseFilterPlanUtil.java | 483 + .../hive/metastore/hbase/TestHBaseStore.java | 1307 + .../metastore/hbase/TestHBaseStoreCached.java | 378 + .../hbase/TestSharedStorageDescriptor.java | 153 + pom.xml | 25 +- .../hadoop/hive/ql/plan/api/Adjacency.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Graph.java | 2 +- .../hadoop/hive/ql/plan/api/Operator.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Query.java | 2 +- .../hadoop/hive/ql/plan/api/QueryPlan.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Stage.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Task.java | 2 +- .../java/org/apache/hadoop/hive/ql/Driver.java | 17 +- .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 333 +- .../apache/hadoop/hive/ql/exec/ExplainTask.java | 24 +- .../hadoop/hive/ql/exec/KeyWrapperFactory.java | 4 + .../hadoop/hive/ql/exec/MapJoinOperator.java | 8 +- .../apache/hadoop/hive/ql/exec/MoveTask.java | 31 +- .../apache/hadoop/hive/ql/exec/Operator.java | 3 +- .../ql/exec/SparkHashTableSinkOperator.java | 3 +- .../hadoop/hive/ql/exec/StatsNoJobTask.java | 25 +- .../apache/hadoop/hive/ql/exec/StatsTask.java | 13 +- .../apache/hadoop/hive/ql/exec/Utilities.java | 11 +- .../persistence/BytesBytesMultiHashMap.java | 11 +- .../persistence/HybridHashTableContainer.java | 68 +- .../ql/exec/persistence/PTFRowContainer.java | 14 +- .../hive/ql/exec/persistence/RowContainer.java | 12 +- .../hadoop/hive/ql/exec/spark/SparkPlan.java | 3 +- .../hive/ql/exec/spark/SparkPlanGenerator.java | 3 +- .../hive/ql/exec/spark/SparkRecordHandler.java | 3 +- .../hadoop/hive/ql/exec/spark/SparkTask.java | 2 +- .../ql/exec/spark/status/SparkJobMonitor.java | 2 +- .../hadoop/hive/ql/exec/tez/DagUtils.java | 3 + .../hive/ql/exec/tez/HashTableLoader.java | 7 +- .../hadoop/hive/ql/exec/tez/InPlaceUpdates.java | 65 + .../hive/ql/exec/tez/RecordProcessor.java | 3 +- .../hive/ql/exec/tez/ReduceRecordProcessor.java | 1 - .../hive/ql/exec/tez/ReduceRecordSource.java | 3 +- .../hadoop/hive/ql/exec/tez/TezJobMonitor.java | 70 +- .../hadoop/hive/ql/exec/tez/TezProcessor.java | 3 +- .../apache/hadoop/hive/ql/exec/tez/TezTask.java | 20 +- .../ql/exec/tez/tools/KeyValuesInputMerger.java | 1 - .../ql/exec/vector/VectorizationContext.java | 10 +- .../apache/hadoop/hive/ql/hooks/ATSHook.java | 9 +- .../hadoop/hive/ql/hooks/LineageLogger.java | 95 +- .../hive/ql/io/CombineHiveInputFormat.java | 10 +- .../hadoop/hive/ql/io/HiveInputFormat.java | 5 +- .../hadoop/hive/ql/io/orc/OrcInputFormat.java | 151 +- .../apache/hadoop/hive/ql/io/orc/OrcSerde.java | 1 + .../apache/hadoop/hive/ql/io/orc/OrcStruct.java | 2 +- .../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 4 +- .../hive/ql/io/parquet/ProjectionPusher.java | 3 +- .../serde/ParquetHiveArrayInspector.java | 12 + .../ql/io/parquet/timestamp/NanoTimeUtils.java | 23 +- .../hive/ql/io/sarg/ConvertAstToSearchArg.java | 4 + .../apache/hadoop/hive/ql/lib/RuleRegExp.java | 61 +- .../apache/hadoop/hive/ql/log/PerfLogger.java | 195 - .../apache/hadoop/hive/ql/metadata/Hive.java | 34 +- .../hadoop/hive/ql/metadata/Partition.java | 29 +- .../ql/optimizer/ColumnPrunerProcFactory.java | 3 + .../hive/ql/optimizer/ConvertJoinMapJoin.java | 22 +- .../hive/ql/optimizer/GenMapRedUtils.java | 57 +- .../ql/optimizer/ReduceSinkMapJoinProc.java | 19 +- .../calcite/reloperators/HiveBetween.java | 75 + .../optimizer/calcite/reloperators/HiveIn.java | 41 + .../calcite/reloperators/HiveLimit.java | 57 - .../calcite/reloperators/HiveSort.java | 110 - .../calcite/reloperators/HiveSortLimit.java | 110 + .../rules/HiveAggregateProjectMergeRule.java | 151 + .../calcite/rules/HivePreFilteringRule.java | 37 +- .../calcite/rules/HiveRelFieldTrimmer.java | 145 +- .../calcite/stats/HiveRelMdMemory.java | 9 +- .../calcite/stats/HiveRelMdParallelism.java | 4 +- .../calcite/translator/ASTConverter.java | 24 +- .../calcite/translator/HiveOpConverter.java | 8 +- .../translator/PlanModifierForASTConv.java | 14 +- .../translator/PlanModifierForReturnPath.java | 4 - .../calcite/translator/PlanModifierUtil.java | 4 +- .../translator/SqlFunctionConverter.java | 16 +- .../hive/ql/optimizer/lineage/LineageCtx.java | 8 +- .../hive/ql/optimizer/ppr/PartitionPruner.java | 7 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 28 +- .../hive/ql/parse/DDLSemanticAnalyzer.java | 17 + .../apache/hadoop/hive/ql/parse/HiveParser.g | 9 +- .../apache/hadoop/hive/ql/parse/QBSubQuery.java | 7 - .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 4 +- .../hive/ql/parse/SemanticAnalyzerFactory.java | 2 + .../hadoop/hive/ql/parse/SubQueryUtils.java | 11 - .../hive/ql/parse/spark/SparkCompiler.java | 3 +- .../org/apache/hadoop/hive/ql/plan/DDLWork.java | 21 + .../hadoop/hive/ql/plan/HiveOperation.java | 1 + .../hive/ql/plan/ShowCreateDatabaseDesc.java | 94 + .../AuthorizationPreEventListener.java | 2 +- .../authorization/plugin/HiveOperationType.java | 1 + .../plugin/sqlstd/Operation2Privilege.java | 2 + .../sqlstd/SQLStdHiveAccessController.java | 5 + .../hadoop/hive/ql/session/SessionState.java | 37 +- .../apache/hadoop/hive/ql/stats/StatsUtils.java | 87 +- .../org/apache/hadoop/hive/ql/udf/UDFJson.java | 2 + .../hive/ql/udf/generic/GenericUDAFMax.java | 16 +- ql/src/main/resources/hive-exec-log4j2.xml | 5 +- ql/src/main/resources/tez-container-log4j2.xml | 5 +- .../hive/metastore/TestMetastoreExpr.java | 2 +- .../persistence/TestBytesBytesMultiHashMap.java | 3 + .../ql/exec/persistence/TestHashPartition.java | 29 + .../exec/persistence/TestPTFRowContainer.java | 31 +- .../hadoop/hive/ql/io/orc/TestOrcStruct.java | 2 + .../serde/TestParquetTimestampUtils.java | 38 +- ...nMapRedUtilsUsePartitionColumnsNegative.java | 73 + ...nMapRedUtilsUsePartitionColumnsPositive.java | 61 + .../authorization_set_show_current_role.q | 3 + .../clientpositive/bucket_map_join_tez1.q | 31 + .../queries/clientpositive/cbo_rp_auto_join17.q | 14 + .../cbo_rp_cross_product_check_2.q | 31 + .../clientpositive/drop_table_with_index.q | 35 + .../test/queries/clientpositive/dynpart_merge.q | 28 + .../dynpart_sort_opt_vectorization.q | 2 + .../clientpositive/dynpart_sort_optimization.q | 2 + .../queries/clientpositive/exchgpartition2lel.q | 32 + ql/src/test/queries/clientpositive/lineage3.q | 26 + .../test/queries/clientpositive/load_orc_part.q | 5 + .../parquet_mixed_partition_formats.q | 42 + .../clientpositive/parquet_ppd_boolean.q | 42 +- .../queries/clientpositive/parquet_ppd_char.q | 46 +- .../queries/clientpositive/parquet_ppd_date.q | 64 +- .../clientpositive/parquet_ppd_decimal.q | 106 +- .../clientpositive/parquet_ppd_timestamp.q | 62 +- .../clientpositive/parquet_ppd_varchar.q | 46 +- .../clientpositive/parquet_predicate_pushdown.q | 20 +- .../clientpositive/show_create_database.q | 3 + .../queries/clientpositive/subquery_views.q | 22 +- .../queries/clientpositive/union_fast_stats.q | 68 + .../queries/clientpositive/vector_char_cast.q | 9 + .../queries/clientpositive/windowing_udaf.q | 4 + .../subquery_exists_implicit_gby.q.out | 8 +- .../subquery_nested_subquery.q.out | 4 +- .../subquery_notexists_implicit_gby.q.out | 8 +- .../subquery_windowing_corr.q.out | 7 +- .../alter_partition_coltype.q.out | 8 +- .../clientpositive/annotate_stats_groupby.q.out | 106 +- .../annotate_stats_groupby2.q.out | 28 +- .../authorization_explain.q.java1.7.out | 2 +- .../authorization_explain.q.java1.8.out | 2 +- .../authorization_set_show_current_role.q.out | 8 + .../results/clientpositive/auto_join18.q.out | 12 +- .../auto_join18_multi_distinct.q.out | 12 +- .../results/clientpositive/auto_join27.q.out | 18 +- .../results/clientpositive/auto_join32.q.out | 4 +- .../clientpositive/binarysortable_1.q.out | Bin 4329 -> 4325 bytes .../clientpositive/cbo_rp_auto_join17.q.out | 118 + .../cbo_rp_cross_product_check_2.q.out | 699 + .../clientpositive/correlationoptimizer2.q.out | 220 +- .../clientpositive/correlationoptimizer6.q.out | 232 +- ql/src/test/results/clientpositive/count.q.out | 14 +- .../results/clientpositive/ctas_colname.q.out | 52 +- .../test/results/clientpositive/database.q.out | 2 +- .../clientpositive/decimal_precision.q.out | 4 +- .../results/clientpositive/decimal_udf.q.out | 30 +- .../results/clientpositive/distinct_stats.q.out | 14 +- .../clientpositive/drop_table_with_index.q.out | 152 + .../results/clientpositive/dynpart_merge.q.out | 99 + .../dynpart_sort_opt_vectorization.q.out | 117 +- .../dynpart_sort_optimization.q.out | 117 +- ...ryption_select_read_only_encrypted_tbl.q.out | 4 +- .../clientpositive/exchgpartition2lel.q.out | 182 + .../clientpositive/explain_dependency.q.out | 18 +- .../clientpositive/explain_dependency2.q.out | 16 +- .../clientpositive/explain_logical.q.out | 78 +- .../clientpositive/fetch_aggregation.q.out | 4 +- .../test/results/clientpositive/gby_star.q.out | 54 +- .../test/results/clientpositive/groupby12.q.out | 6 +- .../results/clientpositive/groupby5_map.q.out | 4 +- .../clientpositive/groupby5_map_skew.q.out | 4 +- .../results/clientpositive/groupby_cube1.q.out | 12 +- .../groupby_distinct_samekey.q.out | 6 +- .../clientpositive/groupby_grouping_sets2.q.out | 10 +- .../clientpositive/groupby_grouping_sets3.q.out | 12 +- .../clientpositive/groupby_grouping_sets5.q.out | 8 +- .../clientpositive/groupby_grouping_sets6.q.out | 8 +- .../clientpositive/groupby_position.q.out | 36 +- .../clientpositive/groupby_resolution.q.out | 60 +- .../clientpositive/groupby_rollup1.q.out | 12 +- .../clientpositive/groupby_sort_10.q.out | 8 +- .../clientpositive/groupby_sort_11.q.out | 10 +- .../results/clientpositive/groupby_sort_8.q.out | 12 +- ql/src/test/results/clientpositive/having.q.out | 62 +- .../test/results/clientpositive/having2.q.out | 12 +- .../clientpositive/index_auto_mult_tables.q.out | 12 +- .../clientpositive/index_auto_self_join.q.out | 12 +- .../clientpositive/index_auto_update.q.out | 6 +- .../index_bitmap_auto_partitioned.q.out | 6 +- .../index_bitmap_compression.q.out | 6 +- .../infer_bucket_sort_dyn_part.q.out | 4 +- .../infer_bucket_sort_map_operators.q.out | 4 +- .../results/clientpositive/input4.q.java1.7.out | 2 +- .../results/clientpositive/input4.q.java1.8.out | 2 +- .../results/clientpositive/join0.q.java1.7.out | 2 +- .../results/clientpositive/join0.q.java1.8.out | 4 +- ql/src/test/results/clientpositive/join18.q.out | 12 +- .../clientpositive/join18_multi_distinct.q.out | 12 +- ql/src/test/results/clientpositive/join31.q.out | 36 +- .../limit_partition_metadataonly.q.out | 4 +- .../results/clientpositive/limit_pushdown.q.out | 36 +- .../test/results/clientpositive/lineage2.q.out | 2 +- .../test/results/clientpositive/lineage3.q.out | 72 +- .../list_bucket_dml_6.q.java1.7.out | 12 +- .../list_bucket_dml_6.q.java1.8.out | 12 +- .../clientpositive/list_bucket_dml_7.q.out | 12 +- .../list_bucket_query_multiskew_3.q.out | 2 +- .../results/clientpositive/load_orc_part.q.out | 26 + .../clientpositive/mapjoin_mapjoin.q.out | 32 +- .../clientpositive/metadata_only_queries.q.out | 4 +- .../results/clientpositive/metadataonly1.q.out | 112 +- .../results/clientpositive/multiMapJoin2.q.out | 226 +- .../nonblock_op_deduplicate.q.out | 8 +- .../results/clientpositive/nonmr_fetch.q.out | 14 +- .../results/clientpositive/parallel_join0.q.out | 2 +- .../parquet_mixed_partition_formats.q.out | 303 + .../clientpositive/parquet_ppd_boolean.q.out | 194 +- .../clientpositive/parquet_ppd_char.q.out | 224 +- .../clientpositive/parquet_ppd_date.q.out | 324 +- .../clientpositive/parquet_ppd_decimal.q.out | 594 +- .../clientpositive/parquet_ppd_timestamp.q.out | 314 +- .../clientpositive/parquet_ppd_varchar.q.out | 224 +- .../parquet_predicate_pushdown.q.out | 118 +- .../clientpositive/partition_multilevels.q.out | 8 +- .../clientpositive/plan_json.q.java1.7.out | 2 +- .../clientpositive/plan_json.q.java1.8.out | 2 +- .../test/results/clientpositive/ppd_gby.q.out | 12 +- .../test/results/clientpositive/ppd_gby2.q.out | 60 +- .../clientpositive/ppd_join_filter.q.out | 98 +- .../ql_rewrite_gbtoidx_cbo_1.q.out | 168 +- .../ql_rewrite_gbtoidx_cbo_2.q.out | 94 +- .../reduce_deduplicate_extended.q.out | 32 +- .../clientpositive/selectDistinctStar.q.out | 44 +- .../clientpositive/show_create_database.q.out | 19 + .../clientpositive/spark/auto_join18.q.out | 10 +- .../spark/auto_join18_multi_distinct.q.out | 12 +- .../clientpositive/spark/auto_join27.q.out | 18 +- .../clientpositive/spark/auto_join32.q.out | 53 +- .../spark/bucket_map_join_tez1.q.out | 357 + .../results/clientpositive/spark/count.q.out | 14 +- .../clientpositive/spark/groupby5_map.q.out | 4 +- .../spark/groupby5_map_skew.q.out | 4 +- .../clientpositive/spark/groupby_cube1.q.out | 12 +- .../clientpositive/spark/groupby_position.q.out | 18 +- .../spark/groupby_resolution.q.out | 60 +- .../clientpositive/spark/groupby_rollup1.q.out | 12 +- .../results/clientpositive/spark/having.q.out | 62 +- .../spark/infer_bucket_sort_map_operators.q.out | 4 +- .../results/clientpositive/spark/join18.q.out | 10 +- .../spark/join18_multi_distinct.q.out | 12 +- .../results/clientpositive/spark/join31.q.out | 36 +- .../spark/limit_partition_metadataonly.q.out | 4 +- .../clientpositive/spark/limit_pushdown.q.out | 34 +- .../clientpositive/spark/mapjoin_mapjoin.q.out | 24 +- .../spark/metadata_only_queries.q.out | 4 +- .../clientpositive/spark/ppd_join_filter.q.out | 90 +- .../spark/ql_rewrite_gbtoidx_cbo_1.q.out | 168 +- .../clientpositive/spark/stats_only_null.q.out | 8 +- .../clientpositive/spark/subquery_in.q.out | 36 +- .../results/clientpositive/spark/union11.q.out | 42 +- .../results/clientpositive/spark/union14.q.out | 28 +- .../results/clientpositive/spark/union15.q.out | 28 +- .../results/clientpositive/spark/union28.q.out | 4 +- .../results/clientpositive/spark/union30.q.out | 4 +- .../results/clientpositive/spark/union33.q.out | 8 +- .../results/clientpositive/spark/union5.q.out | 34 +- .../results/clientpositive/spark/union7.q.out | 28 +- .../clientpositive/spark/union_remove_21.q.out | 4 +- .../spark/vector_count_distinct.q.out | 4 +- .../spark/vector_decimal_aggregate.q.out | 12 +- .../spark/vector_distinct_2.q.out | 28 +- .../clientpositive/spark/vector_groupby_3.q.out | 30 +- .../spark/vector_mapjoin_reduce.q.out | 36 +- .../clientpositive/spark/vector_orderby_5.q.out | 6 +- .../clientpositive/spark/vectorization_0.q.out | 16 +- .../clientpositive/spark/vectorization_13.q.out | 32 +- .../clientpositive/spark/vectorization_15.q.out | 16 +- .../clientpositive/spark/vectorization_16.q.out | 16 +- .../clientpositive/spark/vectorization_9.q.out | 16 +- .../spark/vectorization_pushdown.q.out | 4 +- .../spark/vectorization_short_regress.q.out | 74 +- .../spark/vectorized_nested_mapjoin.q.out | 18 +- .../spark/vectorized_timestamp_funcs.q.out | 12 +- .../clientpositive/stats_only_null.q.out | 8 +- .../results/clientpositive/stats_ppr_all.q.out | 16 +- .../subq_where_serialization.q.out | 18 +- .../clientpositive/subquery_exists_having.q.out | 48 +- .../results/clientpositive/subquery_in.q.out | 36 +- .../clientpositive/subquery_in_having.q.out | 260 +- .../clientpositive/subquery_notexists.q.out | 18 +- .../subquery_notexists_having.q.out | 26 +- .../results/clientpositive/subquery_notin.q.out | 24 +- .../subquery_notin_having.q.java1.7.out | 50 +- .../subquery_unqualcolumnrefs.q.out | 74 +- .../results/clientpositive/subquery_views.q.out | 124 +- .../tez/bucket_map_join_tez1.q.out | 333 + .../clientpositive/tez/constprog_dpp.q.out | 4 +- .../test/results/clientpositive/tez/count.q.out | 14 +- .../tez/dynamic_partition_pruning.q.out | 88 +- .../tez/dynpart_sort_opt_vectorization.q.out | 102 +- .../tez/dynpart_sort_optimization.q.out | 101 +- .../clientpositive/tez/explainuser_1.q.out | 2799 +- .../clientpositive/tez/explainuser_2.q.out | 4004 +- .../clientpositive/tez/explainuser_3.q.out | 10 +- .../results/clientpositive/tez/having.q.out | 62 +- .../clientpositive/tez/limit_pushdown.q.out | 34 +- .../clientpositive/tez/mapjoin_mapjoin.q.out | 24 +- .../tez/metadata_only_queries.q.out | 4 +- .../clientpositive/tez/metadataonly1.q.out | 44 +- .../test/results/clientpositive/tez/mrr.q.out | 94 +- .../clientpositive/tez/selectDistinctStar.q.out | 44 +- .../tez/show_create_database.q.out | 19 + .../clientpositive/tez/stats_only_null.q.out | 8 +- .../clientpositive/tez/subquery_in.q.out | 36 +- .../results/clientpositive/tez/tez_dml.q.out | 6 +- .../results/clientpositive/tez/union5.q.out | 44 +- .../results/clientpositive/tez/union7.q.out | 28 +- .../clientpositive/tez/unionDistinct_1.q.out | 8 +- .../clientpositive/tez/union_fast_stats.q.out | 526 + .../clientpositive/tez/vector_aggregate_9.q.out | 4 +- .../tez/vector_binary_join_groupby.q.out | 4 +- .../clientpositive/tez/vector_char_cast.q.out | 35 + .../tez/vector_count_distinct.q.out | 4 +- .../tez/vector_decimal_aggregate.q.out | 12 +- .../tez/vector_decimal_precision.q.out | 4 +- .../clientpositive/tez/vector_decimal_udf.q.out | 30 +- .../clientpositive/tez/vector_distinct_2.q.out | 28 +- .../clientpositive/tez/vector_groupby_3.q.out | 30 +- .../tez/vector_groupby_reduce.q.out | 8 +- .../tez/vector_grouping_sets.q.out | 8 +- .../tez/vector_mapjoin_reduce.q.out | 36 +- .../clientpositive/tez/vector_orderby_5.q.out | 6 +- .../clientpositive/tez/vector_outer_join2.q.out | 20 +- .../tez/vector_partition_diff_num_cols.q.out | 20 +- .../tez/vector_partitioned_date_time.q.out | 12 +- .../tez/vector_reduce_groupby_decimal.q.out | 24 +- .../clientpositive/tez/vectorization_0.q.out | 16 +- .../clientpositive/tez/vectorization_13.q.out | 32 +- .../clientpositive/tez/vectorization_15.q.out | 16 +- .../clientpositive/tez/vectorization_16.q.out | 16 +- .../clientpositive/tez/vectorization_9.q.out | 16 +- .../tez/vectorization_limit.q.out | 14 +- .../tez/vectorization_pushdown.q.out | 4 +- .../tez/vectorization_short_regress.q.out | 74 +- .../tez/vectorized_distinct_gby.q.out | 8 +- .../vectorized_dynamic_partition_pruning.q.out | 88 +- .../tez/vectorized_nested_mapjoin.q.out | 18 +- .../clientpositive/tez/vectorized_parquet.q.out | 6 +- .../tez/vectorized_timestamp_funcs.q.out | 12 +- ql/src/test/results/clientpositive/udf8.q.out | 4 +- .../test/results/clientpositive/udf_count.q.out | 16 +- .../test/results/clientpositive/union11.q.out | 70 +- .../test/results/clientpositive/union14.q.out | 32 +- .../test/results/clientpositive/union15.q.out | 38 +- .../test/results/clientpositive/union28.q.out | 8 +- .../test/results/clientpositive/union30.q.out | 8 +- .../test/results/clientpositive/union33.q.out | 8 +- ql/src/test/results/clientpositive/union5.q.out | 48 +- ql/src/test/results/clientpositive/union7.q.out | 32 +- .../clientpositive/unionDistinct_1.q.out | 8 +- .../clientpositive/union_fast_stats.q.out | 526 + .../clientpositive/union_remove_21.q.out | 8 +- .../clientpositive/vector_aggregate_9.q.out | 4 +- .../vector_aggregate_without_gby.q.out | 4 +- .../vector_binary_join_groupby.q.out | 4 +- .../clientpositive/vector_char_cast.q.out | 35 + .../clientpositive/vector_count_distinct.q.out | 6 +- .../vector_decimal_aggregate.q.out | 12 +- .../vector_decimal_precision.q.out | 4 +- .../clientpositive/vector_decimal_udf.q.out | 30 +- .../clientpositive/vector_distinct_2.q.out | 28 +- .../clientpositive/vector_groupby_3.q.out | 30 +- .../clientpositive/vector_groupby_reduce.q.out | 8 +- .../clientpositive/vector_grouping_sets.q.out | 8 +- .../clientpositive/vector_left_outer_join.q.out | 8 +- .../clientpositive/vector_mapjoin_reduce.q.out | 36 +- .../clientpositive/vector_orderby_5.q.out | 6 +- .../clientpositive/vector_outer_join1.q.out | 8 +- .../clientpositive/vector_outer_join2.q.out | 28 +- .../clientpositive/vector_outer_join3.q.out | 24 +- .../clientpositive/vector_outer_join4.q.out | 8 +- .../clientpositive/vector_outer_join5.q.out | 48 +- .../vector_partition_diff_num_cols.q.out | 20 +- .../vector_partitioned_date_time.q.out | 12 +- .../vector_reduce_groupby_decimal.q.out | 24 +- .../clientpositive/vectorization_0.q.out | 16 +- .../clientpositive/vectorization_13.q.out | 32 +- .../clientpositive/vectorization_15.q.out | 16 +- .../clientpositive/vectorization_16.q.out | 16 +- .../clientpositive/vectorization_9.q.out | 16 +- .../clientpositive/vectorization_limit.q.out | 16 +- .../clientpositive/vectorization_pushdown.q.out | 4 +- .../vectorization_short_regress.q.out | 74 +- .../vectorized_distinct_gby.q.out | 12 +- .../vectorized_nested_mapjoin.q.out | 26 +- .../clientpositive/vectorized_parquet.q.out | 6 +- .../vectorized_parquet_types.q.out | 6 +- .../vectorized_timestamp_funcs.q.out | 12 +- .../results/clientpositive/windowing_udaf.q.out | 12 + ql/src/test/templates/TestCliDriver.vm | 3 +- .../hadoop/hive/serde/test/InnerStruct.java | 2 +- .../hadoop/hive/serde/test/ThriftTestObj.java | 2 +- .../hadoop/hive/serde2/thrift/test/Complex.java | 2 +- .../hive/serde2/thrift/test/IntString.java | 2 +- .../hive/serde2/thrift/test/MegaStruct.java | 2 +- .../hive/serde2/thrift/test/MiniStruct.java | 2 +- .../hive/serde2/thrift/test/SetIntString.java | 2 +- .../hive/serde2/ColumnProjectionUtils.java | 22 + .../apache/hadoop/hive/serde2/WriteBuffers.java | 10 +- .../BinarySortableSerDeWithEndPrefix.java | 41 + .../hadoop/hive/serde2/lazy/LazyByte.java | 4 + .../hadoop/hive/serde2/lazy/LazyDouble.java | 4 + .../hadoop/hive/serde2/lazy/LazyFloat.java | 4 + .../hadoop/hive/serde2/lazy/LazyInteger.java | 4 + .../hadoop/hive/serde2/lazy/LazyLong.java | 4 + .../hadoop/hive/serde2/lazy/LazyShort.java | 4 + .../hadoop/hive/serde2/lazy/LazyUtils.java | 28 + .../hadoop/hive/service/HiveClusterStatus.java | 2 +- .../hive/service/HiveServerException.java | 2 +- .../apache/hadoop/hive/service/ThriftHive.java | 2 +- .../service/cli/thrift/TArrayTypeEntry.java | 2 +- .../hive/service/cli/thrift/TBinaryColumn.java | 2 +- .../hive/service/cli/thrift/TBoolColumn.java | 2 +- .../hive/service/cli/thrift/TBoolValue.java | 2 +- .../hive/service/cli/thrift/TByteColumn.java | 2 +- .../hive/service/cli/thrift/TByteValue.java | 2 +- .../hive/service/cli/thrift/TCLIService.java | 2 +- .../cli/thrift/TCancelDelegationTokenReq.java | 2 +- .../cli/thrift/TCancelDelegationTokenResp.java | 2 +- .../service/cli/thrift/TCancelOperationReq.java | 2 +- .../cli/thrift/TCancelOperationResp.java | 2 +- .../service/cli/thrift/TCloseOperationReq.java | 2 +- .../service/cli/thrift/TCloseOperationResp.java | 2 +- .../service/cli/thrift/TCloseSessionReq.java | 2 +- .../service/cli/thrift/TCloseSessionResp.java | 2 +- .../hive/service/cli/thrift/TColumnDesc.java | 2 +- .../hive/service/cli/thrift/TDoubleColumn.java | 2 +- .../hive/service/cli/thrift/TDoubleValue.java | 2 +- .../cli/thrift/TExecuteStatementReq.java | 2 +- .../cli/thrift/TExecuteStatementResp.java | 2 +- .../service/cli/thrift/TFetchResultsReq.java | 2 +- .../service/cli/thrift/TFetchResultsResp.java | 2 +- .../service/cli/thrift/TGetCatalogsReq.java | 2 +- .../service/cli/thrift/TGetCatalogsResp.java | 2 +- .../hive/service/cli/thrift/TGetColumnsReq.java | 2 +- .../service/cli/thrift/TGetColumnsResp.java | 2 +- .../cli/thrift/TGetDelegationTokenReq.java | 2 +- .../cli/thrift/TGetDelegationTokenResp.java | 2 +- .../service/cli/thrift/TGetFunctionsReq.java | 2 +- .../service/cli/thrift/TGetFunctionsResp.java | 2 +- .../hive/service/cli/thrift/TGetInfoReq.java | 2 +- .../hive/service/cli/thrift/TGetInfoResp.java | 2 +- .../cli/thrift/TGetOperationStatusReq.java | 2 +- .../cli/thrift/TGetOperationStatusResp.java | 2 +- .../cli/thrift/TGetResultSetMetadataReq.java | 2 +- .../cli/thrift/TGetResultSetMetadataResp.java | 2 +- .../hive/service/cli/thrift/TGetSchemasReq.java | 2 +- .../service/cli/thrift/TGetSchemasResp.java | 2 +- .../service/cli/thrift/TGetTableTypesReq.java | 2 +- .../service/cli/thrift/TGetTableTypesResp.java | 2 +- .../hive/service/cli/thrift/TGetTablesReq.java | 2 +- .../hive/service/cli/thrift/TGetTablesResp.java | 2 +- .../service/cli/thrift/TGetTypeInfoReq.java | 2 +- .../service/cli/thrift/TGetTypeInfoResp.java | 2 +- .../service/cli/thrift/THandleIdentifier.java | 2 +- .../hive/service/cli/thrift/TI16Column.java | 2 +- .../hive/service/cli/thrift/TI16Value.java | 2 +- .../hive/service/cli/thrift/TI32Column.java | 2 +- .../hive/service/cli/thrift/TI32Value.java | 2 +- .../hive/service/cli/thrift/TI64Column.java | 2 +- .../hive/service/cli/thrift/TI64Value.java | 2 +- .../hive/service/cli/thrift/TMapTypeEntry.java | 2 +- .../service/cli/thrift/TOpenSessionReq.java | 2 +- .../service/cli/thrift/TOpenSessionResp.java | 2 +- .../service/cli/thrift/TOperationHandle.java | 2 +- .../service/cli/thrift/TPrimitiveTypeEntry.java | 2 +- .../cli/thrift/TRenewDelegationTokenReq.java | 2 +- .../cli/thrift/TRenewDelegationTokenResp.java | 2 +- .../apache/hive/service/cli/thrift/TRow.java | 2 +- .../apache/hive/service/cli/thrift/TRowSet.java | 2 +- .../hive/service/cli/thrift/TSessionHandle.java | 2 +- .../apache/hive/service/cli/thrift/TStatus.java | 2 +- .../hive/service/cli/thrift/TStringColumn.java | 2 +- .../hive/service/cli/thrift/TStringValue.java | 2 +- .../service/cli/thrift/TStructTypeEntry.java | 2 +- .../hive/service/cli/thrift/TTableSchema.java | 2 +- .../hive/service/cli/thrift/TTypeDesc.java | 2 +- .../service/cli/thrift/TTypeQualifiers.java | 2 +- .../service/cli/thrift/TUnionTypeEntry.java | 2 +- .../cli/thrift/TUserDefinedTypeEntry.java | 2 +- .../gen-py/hive_service/ThriftHive-remote | 49 +- .../auth/LdapAuthenticationProviderImpl.java | 82 +- .../org/apache/hive/service/cli/Column.java | 2 +- .../thrift/RetryingThriftCLIServiceClient.java | 331 + .../org/apache/hive/service/cli/TestColumn.java | 129 + .../cli/TestRetryingThriftCLIServiceClient.java | 133 + .../hadoop/hive/shims/Hadoop20SShims.java | 5 +- .../apache/hadoop/hive/shims/Hadoop23Shims.java | 4 +- .../hive/thrift/HadoopThriftAuthBridge.java | 3 + .../hive/ql/io/sarg/SearchArgumentFactory.java | 5 +- .../hive/ql/io/sarg/SearchArgumentImpl.java | 7 +- testutils/ptest2/src/main/resources/log4j2.xml | 5 +- 721 files changed, 90748 insertions(+), 16405 deletions(-) ----------------------------------------------------------------------
