HIVE-11921 : merge master into branch (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f84eefa5 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f84eefa5 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f84eefa5 Branch: refs/heads/master Commit: f84eefa526346b7b48adebfd48044df7aff04e29 Parents: 4718761 451381c Author: Sergey Shelukhin <[email protected]> Authored: Tue Sep 22 14:31:29 2015 -0700 Committer: Sergey Shelukhin <[email protected]> Committed: Tue Sep 22 14:31:29 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ant/QTestGenTask.java | 11 + bin/ext/hbaseimport.cmd | 35 + bin/ext/hbaseimport.sh | 27 + bin/ext/hbaseschematool.sh | 27 + .../apache/hadoop/hive/common/ObjectPair.java | 5 + .../org/apache/hadoop/hive/conf/HiveConf.java | 98 +- .../apache/hive/common/util/BloomFilter.java | 20 +- data/conf/tez/hive-site.xml | 10 + .../antlr4/org/apache/hive/hplsql/Hplsql.g4 | 70 +- .../main/java/org/apache/hive/hplsql/Cmp.java | 314 + .../java/org/apache/hive/hplsql/Column.java | 29 +- .../main/java/org/apache/hive/hplsql/Conn.java | 21 + .../main/java/org/apache/hive/hplsql/Copy.java | 50 +- .../main/java/org/apache/hive/hplsql/Exec.java | 66 +- .../java/org/apache/hive/hplsql/Expression.java | 33 +- .../main/java/org/apache/hive/hplsql/File.java | 18 +- .../main/java/org/apache/hive/hplsql/Meta.java | 28 +- .../main/java/org/apache/hive/hplsql/Query.java | 18 + .../java/org/apache/hive/hplsql/Select.java | 23 +- .../main/java/org/apache/hive/hplsql/Stmt.java | 8 +- .../main/java/org/apache/hive/hplsql/Var.java | 110 +- .../apache/hive/hplsql/functions/Function.java | 6 +- .../hive/hplsql/functions/FunctionMisc.java | 121 + .../org/apache/hive/hplsql/TestHplsqlLocal.java | 18 + .../apache/hive/hplsql/TestHplsqlOffline.java | 5 + hplsql/src/test/queries/db/cmp_row_count.sql | 4 + hplsql/src/test/queries/db/cmp_sum.sql | 3 + hplsql/src/test/queries/db/copy_to_file.sql | 2 + hplsql/src/test/queries/db/copy_to_hdfs.sql | 2 + hplsql/src/test/queries/db/copy_to_table.sql | 2 + hplsql/src/test/queries/db/part_count.sql | 17 + hplsql/src/test/queries/db/part_count_by.sql | 4 + hplsql/src/test/queries/db/schema.sql | 32 + hplsql/src/test/queries/db/select_into.sql | 20 +- hplsql/src/test/queries/db/select_into2.sql | 17 + .../test/queries/local/create_procedure2.sql | 16 + hplsql/src/test/queries/local/if2.sql | 5 + hplsql/src/test/queries/local/include.sql | 2 + hplsql/src/test/queries/local/include_file.sql | 1 + hplsql/src/test/queries/local/mult_div.sql | 8 + hplsql/src/test/queries/offline/select_db2.sql | 5 + .../src/test/results/db/cmp_row_count.out.txt | 12 + hplsql/src/test/results/db/cmp_sum.out.txt | 320 + hplsql/src/test/results/db/copy_to_file.out.txt | 6 + hplsql/src/test/results/db/copy_to_hdfs.out.txt | 4 + .../src/test/results/db/copy_to_table.out.txt | 2 + hplsql/src/test/results/db/part_count.out.txt | 15 + .../src/test/results/db/part_count_by.out.txt | 13 + hplsql/src/test/results/db/select_into.out.txt | 58 +- hplsql/src/test/results/db/select_into2.out.txt | 19 + .../results/local/create_procedure2.out.txt | 10 + hplsql/src/test/results/local/if2.out.txt | 4 + hplsql/src/test/results/local/include.out.txt | 8 + hplsql/src/test/results/local/mult_div.out.txt | 7 + .../src/test/results/offline/select_db2.out.txt | 6 + itests/hive-unit/pom.xml | 35 + .../hadoop/hive/metastore/TestAdminUser.java | 4 +- .../hive/metastore/TestHiveMetaStore.java | 3 + .../metastore/hbase/HBaseIntegrationTests.java | 117 + .../TestHBaseAggrStatsCacheIntegration.java | 691 + .../hive/metastore/hbase/TestHBaseImport.java | 650 + .../metastore/hbase/TestHBaseMetastoreSql.java | 223 + .../hbase/TestHBaseStoreIntegration.java | 1794 + .../hbase/TestStorageDescriptorSharing.java | 191 + itests/qtest/pom.xml | 10 +- itests/util/pom.xml | 32 + .../metastore/hbase/HBaseStoreTestUtil.java | 45 + .../org/apache/hadoop/hive/ql/QTestUtil.java | 53 +- .../hive/jdbc/ZooKeeperHiveClientHelper.java | 32 +- metastore/if/hive_metastore.thrift | 54 + metastore/pom.xml | 82 + .../metastore/hbase/HbaseMetastoreProto.java | 34901 +++++++++++++++++ .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 6919 ++-- .../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 664 + .../ThriftHiveMetastore_server.skeleton.cpp | 25 + .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 1294 +- .../gen/thrift/gen-cpp/hive_metastore_types.h | 371 + .../hive/metastore/api/AbortTxnRequest.java | 2 +- .../metastore/api/AddDynamicPartitions.java | 2 +- .../metastore/api/AddPartitionsRequest.java | 2 +- .../hive/metastore/api/AddPartitionsResult.java | 2 +- .../hadoop/hive/metastore/api/AggrStats.java | 2 +- .../metastore/api/AlreadyExistsException.java | 2 +- .../metastore/api/BinaryColumnStatsData.java | 2 +- .../metastore/api/BooleanColumnStatsData.java | 2 +- .../hive/metastore/api/CheckLockRequest.java | 2 +- .../metastore/api/ClearFileMetadataRequest.java | 438 + .../metastore/api/ClearFileMetadataResult.java | 283 + .../hive/metastore/api/ColumnStatistics.java | 2 +- .../metastore/api/ColumnStatisticsDesc.java | 2 +- .../hive/metastore/api/ColumnStatisticsObj.java | 2 +- .../hive/metastore/api/CommitTxnRequest.java | 2 +- .../hive/metastore/api/CompactionRequest.java | 2 +- .../api/ConfigValSecurityException.java | 2 +- .../api/CurrentNotificationEventId.java | 2 +- .../hadoop/hive/metastore/api/Database.java | 2 +- .../apache/hadoop/hive/metastore/api/Date.java | 2 +- .../hive/metastore/api/DateColumnStatsData.java | 2 +- .../hadoop/hive/metastore/api/Decimal.java | 2 +- .../metastore/api/DecimalColumnStatsData.java | 2 +- .../metastore/api/DoubleColumnStatsData.java | 2 +- .../hive/metastore/api/DropPartitionsExpr.java | 2 +- .../metastore/api/DropPartitionsRequest.java | 2 +- .../metastore/api/DropPartitionsResult.java | 2 +- .../hive/metastore/api/EnvironmentContext.java | 2 +- .../hadoop/hive/metastore/api/FieldSchema.java | 2 +- .../hive/metastore/api/FireEventRequest.java | 2 +- .../hive/metastore/api/FireEventResponse.java | 2 +- .../hadoop/hive/metastore/api/Function.java | 2 +- .../metastore/api/GetAllFunctionsResponse.java | 38 +- .../api/GetFileMetadataByExprRequest.java | 548 + .../api/GetFileMetadataByExprResult.java | 703 + .../metastore/api/GetFileMetadataRequest.java | 438 + .../metastore/api/GetFileMetadataResult.java | 540 + .../metastore/api/GetOpenTxnsInfoResponse.java | 2 +- .../hive/metastore/api/GetOpenTxnsResponse.java | 2 +- .../api/GetPrincipalsInRoleRequest.java | 2 +- .../api/GetPrincipalsInRoleResponse.java | 2 +- .../api/GetRoleGrantsForPrincipalRequest.java | 2 +- .../api/GetRoleGrantsForPrincipalResponse.java | 2 +- .../api/GrantRevokePrivilegeRequest.java | 2 +- .../api/GrantRevokePrivilegeResponse.java | 2 +- .../metastore/api/GrantRevokeRoleRequest.java | 2 +- .../metastore/api/GrantRevokeRoleResponse.java | 2 +- .../hive/metastore/api/HeartbeatRequest.java | 2 +- .../metastore/api/HeartbeatTxnRangeRequest.java | 2 +- .../api/HeartbeatTxnRangeResponse.java | 2 +- .../hive/metastore/api/HiveObjectPrivilege.java | 2 +- .../hive/metastore/api/HiveObjectRef.java | 2 +- .../apache/hadoop/hive/metastore/api/Index.java | 2 +- .../api/IndexAlreadyExistsException.java | 2 +- .../metastore/api/InsertEventRequestData.java | 2 +- .../metastore/api/InvalidInputException.java | 2 +- .../metastore/api/InvalidObjectException.java | 2 +- .../api/InvalidOperationException.java | 2 +- .../api/InvalidPartitionException.java | 2 +- .../hive/metastore/api/LockComponent.java | 2 +- .../hadoop/hive/metastore/api/LockRequest.java | 2 +- .../hadoop/hive/metastore/api/LockResponse.java | 2 +- .../hive/metastore/api/LongColumnStatsData.java | 2 +- .../hive/metastore/api/MetaException.java | 2 +- .../hive/metastore/api/MetadataPpdResult.java | 508 + .../hive/metastore/api/NoSuchLockException.java | 2 +- .../metastore/api/NoSuchObjectException.java | 2 +- .../hive/metastore/api/NoSuchTxnException.java | 2 +- .../hive/metastore/api/NotificationEvent.java | 2 +- .../metastore/api/NotificationEventRequest.java | 2 +- .../api/NotificationEventResponse.java | 2 +- .../hive/metastore/api/OpenTxnRequest.java | 2 +- .../hive/metastore/api/OpenTxnsResponse.java | 2 +- .../apache/hadoop/hive/metastore/api/Order.java | 2 +- .../hadoop/hive/metastore/api/Partition.java | 2 +- .../api/PartitionListComposingSpec.java | 2 +- .../hive/metastore/api/PartitionSpec.java | 2 +- .../api/PartitionSpecWithSharedSD.java | 2 +- .../hive/metastore/api/PartitionWithoutSD.java | 2 +- .../metastore/api/PartitionsByExprRequest.java | 2 +- .../metastore/api/PartitionsByExprResult.java | 2 +- .../metastore/api/PartitionsStatsRequest.java | 2 +- .../metastore/api/PartitionsStatsResult.java | 2 +- .../metastore/api/PrincipalPrivilegeSet.java | 2 +- .../hadoop/hive/metastore/api/PrivilegeBag.java | 2 +- .../hive/metastore/api/PrivilegeGrantInfo.java | 2 +- .../metastore/api/PutFileMetadataRequest.java | 588 + .../metastore/api/PutFileMetadataResult.java | 283 + .../hadoop/hive/metastore/api/ResourceUri.java | 2 +- .../apache/hadoop/hive/metastore/api/Role.java | 2 +- .../hive/metastore/api/RolePrincipalGrant.java | 2 +- .../hadoop/hive/metastore/api/Schema.java | 2 +- .../hadoop/hive/metastore/api/SerDeInfo.java | 2 +- .../api/SetPartitionsStatsRequest.java | 2 +- .../hive/metastore/api/ShowCompactRequest.java | 2 +- .../hive/metastore/api/ShowCompactResponse.java | 2 +- .../api/ShowCompactResponseElement.java | 2 +- .../hive/metastore/api/ShowLocksRequest.java | 2 +- .../hive/metastore/api/ShowLocksResponse.java | 2 +- .../metastore/api/ShowLocksResponseElement.java | 2 +- .../hadoop/hive/metastore/api/SkewedInfo.java | 2 +- .../hive/metastore/api/StorageDescriptor.java | 2 +- .../metastore/api/StringColumnStatsData.java | 2 +- .../apache/hadoop/hive/metastore/api/Table.java | 2 +- .../hive/metastore/api/TableStatsRequest.java | 2 +- .../hive/metastore/api/TableStatsResult.java | 2 +- .../hive/metastore/api/ThriftHiveMetastore.java | 8422 ++-- .../hive/metastore/api/TxnAbortedException.java | 2 +- .../hadoop/hive/metastore/api/TxnInfo.java | 2 +- .../hive/metastore/api/TxnOpenException.java | 2 +- .../apache/hadoop/hive/metastore/api/Type.java | 2 +- .../hive/metastore/api/UnknownDBException.java | 2 +- .../api/UnknownPartitionException.java | 2 +- .../metastore/api/UnknownTableException.java | 2 +- .../hive/metastore/api/UnlockRequest.java | 2 +- .../hadoop/hive/metastore/api/Version.java | 2 +- .../gen-php/metastore/ThriftHiveMetastore.php | 2810 +- .../src/gen/thrift/gen-php/metastore/Types.php | 1009 +- .../hive_metastore/ThriftHiveMetastore-remote | 49 +- .../hive_metastore/ThriftHiveMetastore.py | 1563 +- .../gen/thrift/gen-py/hive_metastore/ttypes.py | 734 +- .../gen/thrift/gen-rb/hive_metastore_types.rb | 167 + .../gen/thrift/gen-rb/thrift_hive_metastore.rb | 267 + .../hadoop/hive/metastore/HiveAlterHandler.java | 38 +- .../hadoop/hive/metastore/HiveMetaStore.java | 266 +- .../hive/metastore/HiveMetaStoreClient.java | 112 +- .../hadoop/hive/metastore/IMetaStoreClient.java | 6 + .../hadoop/hive/metastore/ObjectStore.java | 418 +- .../hive/metastore/PartFilterExprUtil.java | 149 + .../apache/hadoop/hive/metastore/RawStore.java | 66 +- .../hadoop/hive/metastore/RawStoreProxy.java | 5 +- .../hbase/AggrStatsInvalidatorFilter.java | 121 + .../hadoop/hive/metastore/hbase/Counter.java | 53 + .../hive/metastore/hbase/HBaseConnection.java | 96 + .../metastore/hbase/HBaseFilterPlanUtil.java | 612 + .../hive/metastore/hbase/HBaseImport.java | 535 + .../hive/metastore/hbase/HBaseReadWrite.java | 2106 + .../hive/metastore/hbase/HBaseSchemaTool.java | 239 + .../hadoop/hive/metastore/hbase/HBaseStore.java | 2387 ++ .../hadoop/hive/metastore/hbase/HBaseUtils.java | 1340 + .../hive/metastore/hbase/ObjectCache.java | 81 + .../hive/metastore/hbase/PartitionCache.java | 168 + .../metastore/hbase/PartitionKeyComparator.java | 292 + .../hbase/SharedStorageDescriptor.java | 251 + .../hadoop/hive/metastore/hbase/StatsCache.java | 326 + .../metastore/hbase/TephraHBaseConnection.java | 127 + .../metastore/hbase/VanillaHBaseConnection.java | 137 + .../stats/BinaryColumnStatsAggregator.java | 35 + .../stats/BooleanColumnStatsAggregator.java | 35 + .../hbase/stats/ColumnStatsAggregator.java | 26 + .../stats/ColumnStatsAggregatorFactory.java | 94 + .../stats/DecimalColumnStatsAggregator.java | 43 + .../stats/DoubleColumnStatsAggregator.java | 36 + .../hbase/stats/LongColumnStatsAggregator.java | 36 + .../stats/StringColumnStatsAggregator.java | 36 + .../hive/metastore/parser/ExpressionTree.java | 9 +- .../metastore/hbase/hbase_metastore_proto.proto | 282 + .../DummyRawStoreControlledCommit.java | 56 +- .../DummyRawStoreForJdoConnection.java | 50 +- .../hadoop/hive/metastore/TestObjectStore.java | 43 +- .../hadoop/hive/metastore/hbase/MockUtils.java | 199 + .../hbase/TestHBaseAggregateStatsCache.java | 316 + .../hbase/TestHBaseFilterPlanUtil.java | 483 + .../hive/metastore/hbase/TestHBaseStore.java | 1307 + .../metastore/hbase/TestHBaseStoreCached.java | 378 + .../hbase/TestSharedStorageDescriptor.java | 153 + pom.xml | 3 +- .../hadoop/hive/ql/plan/api/Adjacency.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Graph.java | 2 +- .../hadoop/hive/ql/plan/api/Operator.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Query.java | 2 +- .../hadoop/hive/ql/plan/api/QueryPlan.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Stage.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Task.java | 2 +- .../java/org/apache/hadoop/hive/ql/Driver.java | 6 + .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 64 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 9 +- .../hadoop/hive/ql/metadata/Partition.java | 29 +- .../AuthorizationPreEventListener.java | 2 +- .../apache/hadoop/hive/ql/stats/StatsUtils.java | 3 + .../hive/metastore/TestMetastoreExpr.java | 2 +- .../dynpart_sort_opt_vectorization.q | 2 + .../clientpositive/dynpart_sort_optimization.q | 2 + .../dynpart_sort_opt_vectorization.q.out | 12 +- .../dynpart_sort_optimization.q.out | 12 +- .../tez/dynpart_sort_opt_vectorization.q.out | 12 +- .../tez/dynpart_sort_optimization.q.out | 12 +- ql/src/test/templates/TestCliDriver.vm | 4 +- .../hadoop/hive/serde/test/InnerStruct.java | 2 +- .../hadoop/hive/serde/test/ThriftTestObj.java | 2 +- .../hadoop/hive/serde2/thrift/test/Complex.java | 2 +- .../hive/serde2/thrift/test/IntString.java | 2 +- .../hive/serde2/thrift/test/MegaStruct.java | 2 +- .../hive/serde2/thrift/test/MiniStruct.java | 2 +- .../hive/serde2/thrift/test/SetIntString.java | 2 +- .../BinarySortableSerDeWithEndPrefix.java | 41 + .../hadoop/hive/service/HiveClusterStatus.java | 2 +- .../hive/service/HiveServerException.java | 2 +- .../apache/hadoop/hive/service/ThriftHive.java | 2 +- .../service/cli/thrift/TArrayTypeEntry.java | 2 +- .../hive/service/cli/thrift/TBinaryColumn.java | 2 +- .../hive/service/cli/thrift/TBoolColumn.java | 2 +- .../hive/service/cli/thrift/TBoolValue.java | 2 +- .../hive/service/cli/thrift/TByteColumn.java | 2 +- .../hive/service/cli/thrift/TByteValue.java | 2 +- .../hive/service/cli/thrift/TCLIService.java | 2 +- .../cli/thrift/TCancelDelegationTokenReq.java | 2 +- .../cli/thrift/TCancelDelegationTokenResp.java | 2 +- .../service/cli/thrift/TCancelOperationReq.java | 2 +- .../cli/thrift/TCancelOperationResp.java | 2 +- .../service/cli/thrift/TCloseOperationReq.java | 2 +- .../service/cli/thrift/TCloseOperationResp.java | 2 +- .../service/cli/thrift/TCloseSessionReq.java | 2 +- .../service/cli/thrift/TCloseSessionResp.java | 2 +- .../hive/service/cli/thrift/TColumnDesc.java | 2 +- .../hive/service/cli/thrift/TDoubleColumn.java | 2 +- .../hive/service/cli/thrift/TDoubleValue.java | 2 +- .../cli/thrift/TExecuteStatementReq.java | 2 +- .../cli/thrift/TExecuteStatementResp.java | 2 +- .../service/cli/thrift/TFetchResultsReq.java | 2 +- .../service/cli/thrift/TFetchResultsResp.java | 2 +- .../service/cli/thrift/TGetCatalogsReq.java | 2 +- .../service/cli/thrift/TGetCatalogsResp.java | 2 +- .../hive/service/cli/thrift/TGetColumnsReq.java | 2 +- .../service/cli/thrift/TGetColumnsResp.java | 2 +- .../cli/thrift/TGetDelegationTokenReq.java | 2 +- .../cli/thrift/TGetDelegationTokenResp.java | 2 +- .../service/cli/thrift/TGetFunctionsReq.java | 2 +- .../service/cli/thrift/TGetFunctionsResp.java | 2 +- .../hive/service/cli/thrift/TGetInfoReq.java | 2 +- .../hive/service/cli/thrift/TGetInfoResp.java | 2 +- .../cli/thrift/TGetOperationStatusReq.java | 2 +- .../cli/thrift/TGetOperationStatusResp.java | 2 +- .../cli/thrift/TGetResultSetMetadataReq.java | 2 +- .../cli/thrift/TGetResultSetMetadataResp.java | 2 +- .../hive/service/cli/thrift/TGetSchemasReq.java | 2 +- .../service/cli/thrift/TGetSchemasResp.java | 2 +- .../service/cli/thrift/TGetTableTypesReq.java | 2 +- .../service/cli/thrift/TGetTableTypesResp.java | 2 +- .../hive/service/cli/thrift/TGetTablesReq.java | 2 +- .../hive/service/cli/thrift/TGetTablesResp.java | 2 +- .../service/cli/thrift/TGetTypeInfoReq.java | 2 +- .../service/cli/thrift/TGetTypeInfoResp.java | 2 +- .../service/cli/thrift/THandleIdentifier.java | 2 +- .../hive/service/cli/thrift/TI16Column.java | 2 +- .../hive/service/cli/thrift/TI16Value.java | 2 +- .../hive/service/cli/thrift/TI32Column.java | 2 +- .../hive/service/cli/thrift/TI32Value.java | 2 +- .../hive/service/cli/thrift/TI64Column.java | 2 +- .../hive/service/cli/thrift/TI64Value.java | 2 +- .../hive/service/cli/thrift/TMapTypeEntry.java | 2 +- .../service/cli/thrift/TOpenSessionReq.java | 2 +- .../service/cli/thrift/TOpenSessionResp.java | 2 +- .../service/cli/thrift/TOperationHandle.java | 2 +- .../service/cli/thrift/TPrimitiveTypeEntry.java | 2 +- .../cli/thrift/TRenewDelegationTokenReq.java | 2 +- .../cli/thrift/TRenewDelegationTokenResp.java | 2 +- .../apache/hive/service/cli/thrift/TRow.java | 2 +- .../apache/hive/service/cli/thrift/TRowSet.java | 2 +- .../hive/service/cli/thrift/TSessionHandle.java | 2 +- .../apache/hive/service/cli/thrift/TStatus.java | 2 +- .../hive/service/cli/thrift/TStringColumn.java | 2 +- .../hive/service/cli/thrift/TStringValue.java | 2 +- .../service/cli/thrift/TStructTypeEntry.java | 2 +- .../hive/service/cli/thrift/TTableSchema.java | 2 +- .../hive/service/cli/thrift/TTypeDesc.java | 2 +- .../service/cli/thrift/TTypeQualifiers.java | 2 +- .../service/cli/thrift/TUnionTypeEntry.java | 2 +- .../cli/thrift/TUserDefinedTypeEntry.java | 2 +- .../gen-py/hive_service/ThriftHive-remote | 49 +- 347 files changed, 76375 insertions(+), 7194 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f84eefa5/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f84eefa5/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f84eefa5/data/conf/tez/hive-site.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f84eefa5/itests/qtest/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f84eefa5/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --cc itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 0f17b1b,aab3aab..4cab161 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@@ -53,11 -54,15 +53,12 @@@ import java.util.concurrent.TimeUnit import java.util.regex.Matcher; import java.util.regex.Pattern; -import junit.framework.Assert; -import junit.framework.TestSuite; - -import org.apache.commons.lang.StringUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@@ -347,18 -350,43 +353,44 @@@ public class QTestUtil return "jceks://file" + new Path(keyDir, "test.jks").toUri(); } + private void startMiniHBaseCluster() throws Exception { + Configuration hbaseConf = HBaseConfiguration.create(); + hbaseConf.setInt("hbase.master.info.port", -1); + utility = new HBaseTestingUtility(hbaseConf); + utility.startMiniCluster(); + conf = new HiveConf(utility.getConfiguration(), Driver.class); + HBaseAdmin admin = utility.getHBaseAdmin(); + // Need to use reflection here to make compilation pass since HBaseIntegrationTests + // is not compiled in hadoop-1. All HBaseMetastore tests run under hadoop-2, so this + // guarantee HBaseIntegrationTests exist when we hitting this code path + java.lang.reflect.Method initHBaseMetastoreMethod = Class.forName( + "org.apache.hadoop.hive.metastore.hbase.HBaseStoreTestUtil") + .getMethod("initHBaseMetastore", HBaseAdmin.class, HiveConf.class); + initHBaseMetastoreMethod.invoke(null, admin, conf); + } + public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, - String confDir, String hadoopVer, String initScript, - String cleanupScript) throws Exception { + String confDir, String hadoopVer, String initScript, String cleanupScript) + throws Exception { + this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, false); + } + + public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, + String confDir, String hadoopVer, String initScript, String cleanupScript, boolean useHBaseMetastore) + throws Exception { this.outDir = outDir; this.logDir = logDir; + this.useHBaseMetastore = useHBaseMetastore; + Logger hadoopLog = Logger.getLogger("org.apache.hadoop"); + hadoopLog.setLevel(Level.INFO); if (confDir != null && !confDir.isEmpty()) { - HiveConf.setHiveSiteLocation(new URL("file://" - + new File(confDir).toURI().getPath() + "/hive-site.xml")); + HiveConf.setHiveSiteLocation(new URL("file://"+ new File(confDir).toURI().getPath() + "/hive-site.xml")); System.out.println("Setting hive-site: "+HiveConf.getHiveSiteLocation()); } + if (useHBaseMetastore) { + startMiniHBaseCluster(); + } conf = new HiveConf(Driver.class); this.hadoopVer = getHadoopMainVersion(hadoopVer); qMap = new TreeMap<String, String>(); http://git-wip-us.apache.org/repos/asf/hive/blob/f84eefa5/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f84eefa5/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f84eefa5/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out ----------------------------------------------------------------------
