HIVE-11711: Merge hbase-metastore branch to trunk
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/52383033 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/52383033 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/52383033 Branch: refs/heads/master Commit: 52383033822b87b02853eaaf15db1b2904617615 Parents: 514ab79 4c17ecf Author: Daniel Dai <da...@hortonworks.com> Authored: Mon Sep 21 22:02:22 2015 -0700 Committer: Daniel Dai <da...@hortonworks.com> Committed: Mon Sep 21 22:02:22 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ant/QTestGenTask.java | 11 + bin/ext/hbaseimport.cmd | 35 + bin/ext/hbaseimport.sh | 27 + bin/ext/hbaseschematool.sh | 27 + .../apache/hadoop/hive/common/ObjectPair.java | 5 + .../org/apache/hadoop/hive/conf/HiveConf.java | 98 +- .../apache/hive/common/util/BloomFilter.java | 20 +- data/conf/tez/hive-site.xml | 10 + itests/hive-unit/pom.xml | 35 + .../hadoop/hive/metastore/TestAdminUser.java | 4 +- .../hive/metastore/TestHiveMetaStore.java | 3 + .../metastore/hbase/HBaseIntegrationTests.java | 117 + .../TestHBaseAggrStatsCacheIntegration.java | 691 + .../hive/metastore/hbase/TestHBaseImport.java | 650 + .../metastore/hbase/TestHBaseMetastoreSql.java | 223 + .../hbase/TestHBaseStoreIntegration.java | 1794 + .../hbase/TestStorageDescriptorSharing.java | 191 + itests/qtest/pom.xml | 10 +- itests/util/pom.xml | 32 + .../metastore/hbase/HBaseStoreTestUtil.java | 45 + .../org/apache/hadoop/hive/ql/QTestUtil.java | 41 +- metastore/if/hive_metastore.thrift | 54 + metastore/pom.xml | 82 + .../metastore/hbase/HbaseMetastoreProto.java | 34901 +++++++++++++++++ .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 6919 ++-- .../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 664 + .../ThriftHiveMetastore_server.skeleton.cpp | 25 + .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 1294 +- .../gen/thrift/gen-cpp/hive_metastore_types.h | 371 + .../hive/metastore/api/AbortTxnRequest.java | 2 +- .../metastore/api/AddDynamicPartitions.java | 2 +- .../metastore/api/AddPartitionsRequest.java | 2 +- .../hive/metastore/api/AddPartitionsResult.java | 2 +- .../hadoop/hive/metastore/api/AggrStats.java | 2 +- .../metastore/api/AlreadyExistsException.java | 2 +- .../metastore/api/BinaryColumnStatsData.java | 2 +- .../metastore/api/BooleanColumnStatsData.java | 2 +- .../hive/metastore/api/CheckLockRequest.java | 2 +- .../metastore/api/ClearFileMetadataRequest.java | 438 + .../metastore/api/ClearFileMetadataResult.java | 283 + .../hive/metastore/api/ColumnStatistics.java | 2 +- .../metastore/api/ColumnStatisticsDesc.java | 2 +- .../hive/metastore/api/ColumnStatisticsObj.java | 2 +- .../hive/metastore/api/CommitTxnRequest.java | 2 +- .../hive/metastore/api/CompactionRequest.java | 2 +- .../api/ConfigValSecurityException.java | 2 +- .../api/CurrentNotificationEventId.java | 2 +- .../hadoop/hive/metastore/api/Database.java | 2 +- .../apache/hadoop/hive/metastore/api/Date.java | 2 +- .../hive/metastore/api/DateColumnStatsData.java | 2 +- .../hadoop/hive/metastore/api/Decimal.java | 2 +- .../metastore/api/DecimalColumnStatsData.java | 2 +- .../metastore/api/DoubleColumnStatsData.java | 2 +- .../hive/metastore/api/DropPartitionsExpr.java | 2 +- .../metastore/api/DropPartitionsRequest.java | 2 +- .../metastore/api/DropPartitionsResult.java | 2 +- .../hive/metastore/api/EnvironmentContext.java | 2 +- .../hadoop/hive/metastore/api/FieldSchema.java | 2 +- .../hive/metastore/api/FireEventRequest.java | 2 +- .../hive/metastore/api/FireEventResponse.java | 2 +- .../hadoop/hive/metastore/api/Function.java | 2 +- .../metastore/api/GetAllFunctionsResponse.java | 38 +- .../api/GetFileMetadataByExprRequest.java | 548 + .../api/GetFileMetadataByExprResult.java | 703 + .../metastore/api/GetFileMetadataRequest.java | 438 + .../metastore/api/GetFileMetadataResult.java | 540 + .../metastore/api/GetOpenTxnsInfoResponse.java | 2 +- .../hive/metastore/api/GetOpenTxnsResponse.java | 2 +- .../api/GetPrincipalsInRoleRequest.java | 2 +- .../api/GetPrincipalsInRoleResponse.java | 2 +- .../api/GetRoleGrantsForPrincipalRequest.java | 2 +- .../api/GetRoleGrantsForPrincipalResponse.java | 2 +- .../api/GrantRevokePrivilegeRequest.java | 2 +- .../api/GrantRevokePrivilegeResponse.java | 2 +- .../metastore/api/GrantRevokeRoleRequest.java | 2 +- .../metastore/api/GrantRevokeRoleResponse.java | 2 +- .../hive/metastore/api/HeartbeatRequest.java | 2 +- .../metastore/api/HeartbeatTxnRangeRequest.java | 2 +- .../api/HeartbeatTxnRangeResponse.java | 2 +- .../hive/metastore/api/HiveObjectPrivilege.java | 2 +- .../hive/metastore/api/HiveObjectRef.java | 2 +- .../apache/hadoop/hive/metastore/api/Index.java | 2 +- .../api/IndexAlreadyExistsException.java | 2 +- .../metastore/api/InsertEventRequestData.java | 2 +- .../metastore/api/InvalidInputException.java | 2 +- .../metastore/api/InvalidObjectException.java | 2 +- .../api/InvalidOperationException.java | 2 +- .../api/InvalidPartitionException.java | 2 +- .../hive/metastore/api/LockComponent.java | 2 +- .../hadoop/hive/metastore/api/LockRequest.java | 2 +- .../hadoop/hive/metastore/api/LockResponse.java | 2 +- .../hive/metastore/api/LongColumnStatsData.java | 2 +- .../hive/metastore/api/MetaException.java | 2 +- .../hive/metastore/api/MetadataPpdResult.java | 508 + .../hive/metastore/api/NoSuchLockException.java | 2 +- .../metastore/api/NoSuchObjectException.java | 2 +- .../hive/metastore/api/NoSuchTxnException.java | 2 +- .../hive/metastore/api/NotificationEvent.java | 2 +- .../metastore/api/NotificationEventRequest.java | 2 +- .../api/NotificationEventResponse.java | 2 +- .../hive/metastore/api/OpenTxnRequest.java | 2 +- .../hive/metastore/api/OpenTxnsResponse.java | 2 +- .../apache/hadoop/hive/metastore/api/Order.java | 2 +- .../hadoop/hive/metastore/api/Partition.java | 2 +- .../api/PartitionListComposingSpec.java | 2 +- .../hive/metastore/api/PartitionSpec.java | 2 +- .../api/PartitionSpecWithSharedSD.java | 2 +- .../hive/metastore/api/PartitionWithoutSD.java | 2 +- .../metastore/api/PartitionsByExprRequest.java | 2 +- .../metastore/api/PartitionsByExprResult.java | 2 +- .../metastore/api/PartitionsStatsRequest.java | 2 +- .../metastore/api/PartitionsStatsResult.java | 2 +- .../metastore/api/PrincipalPrivilegeSet.java | 2 +- .../hadoop/hive/metastore/api/PrivilegeBag.java | 2 +- .../hive/metastore/api/PrivilegeGrantInfo.java | 2 +- .../metastore/api/PutFileMetadataRequest.java | 588 + .../metastore/api/PutFileMetadataResult.java | 283 + .../hadoop/hive/metastore/api/ResourceUri.java | 2 +- .../apache/hadoop/hive/metastore/api/Role.java | 2 +- .../hive/metastore/api/RolePrincipalGrant.java | 2 +- .../hadoop/hive/metastore/api/Schema.java | 2 +- .../hadoop/hive/metastore/api/SerDeInfo.java | 2 +- .../api/SetPartitionsStatsRequest.java | 2 +- .../hive/metastore/api/ShowCompactRequest.java | 2 +- .../hive/metastore/api/ShowCompactResponse.java | 2 +- .../api/ShowCompactResponseElement.java | 2 +- .../hive/metastore/api/ShowLocksRequest.java | 2 +- .../hive/metastore/api/ShowLocksResponse.java | 2 +- .../metastore/api/ShowLocksResponseElement.java | 2 +- .../hadoop/hive/metastore/api/SkewedInfo.java | 2 +- .../hive/metastore/api/StorageDescriptor.java | 2 +- .../metastore/api/StringColumnStatsData.java | 2 +- .../apache/hadoop/hive/metastore/api/Table.java | 2 +- .../hive/metastore/api/TableStatsRequest.java | 2 +- .../hive/metastore/api/TableStatsResult.java | 2 +- .../hive/metastore/api/ThriftHiveMetastore.java | 8422 ++-- .../hive/metastore/api/TxnAbortedException.java | 2 +- .../hadoop/hive/metastore/api/TxnInfo.java | 2 +- .../hive/metastore/api/TxnOpenException.java | 2 +- .../apache/hadoop/hive/metastore/api/Type.java | 2 +- .../hive/metastore/api/UnknownDBException.java | 2 +- .../api/UnknownPartitionException.java | 2 +- .../metastore/api/UnknownTableException.java | 2 +- .../hive/metastore/api/UnlockRequest.java | 2 +- .../hadoop/hive/metastore/api/Version.java | 2 +- .../gen-php/metastore/ThriftHiveMetastore.php | 2810 +- .../src/gen/thrift/gen-php/metastore/Types.php | 1009 +- .../hive_metastore/ThriftHiveMetastore-remote | 49 +- .../hive_metastore/ThriftHiveMetastore.py | 1563 +- .../gen/thrift/gen-py/hive_metastore/ttypes.py | 734 +- .../gen/thrift/gen-rb/hive_metastore_types.rb | 167 + .../gen/thrift/gen-rb/thrift_hive_metastore.rb | 267 + .../hadoop/hive/metastore/HiveAlterHandler.java | 38 +- .../hadoop/hive/metastore/HiveMetaStore.java | 266 +- .../hive/metastore/HiveMetaStoreClient.java | 112 +- .../hadoop/hive/metastore/IMetaStoreClient.java | 6 + .../hadoop/hive/metastore/ObjectStore.java | 418 +- .../hive/metastore/PartFilterExprUtil.java | 149 + .../apache/hadoop/hive/metastore/RawStore.java | 66 +- .../hadoop/hive/metastore/RawStoreProxy.java | 5 +- .../hbase/AggrStatsInvalidatorFilter.java | 121 + .../hadoop/hive/metastore/hbase/Counter.java | 53 + .../hive/metastore/hbase/HBaseConnection.java | 96 + .../metastore/hbase/HBaseFilterPlanUtil.java | 612 + .../hive/metastore/hbase/HBaseImport.java | 535 + .../hive/metastore/hbase/HBaseReadWrite.java | 2106 + .../hive/metastore/hbase/HBaseSchemaTool.java | 239 + .../hadoop/hive/metastore/hbase/HBaseStore.java | 2387 ++ .../hadoop/hive/metastore/hbase/HBaseUtils.java | 1340 + .../hive/metastore/hbase/ObjectCache.java | 81 + .../hive/metastore/hbase/PartitionCache.java | 168 + .../metastore/hbase/PartitionKeyComparator.java | 292 + .../hbase/SharedStorageDescriptor.java | 251 + .../hadoop/hive/metastore/hbase/StatsCache.java | 326 + .../metastore/hbase/TephraHBaseConnection.java | 127 + .../metastore/hbase/VanillaHBaseConnection.java | 137 + .../stats/BinaryColumnStatsAggregator.java | 35 + .../stats/BooleanColumnStatsAggregator.java | 35 + .../hbase/stats/ColumnStatsAggregator.java | 26 + .../stats/ColumnStatsAggregatorFactory.java | 94 + .../stats/DecimalColumnStatsAggregator.java | 43 + .../stats/DoubleColumnStatsAggregator.java | 36 + .../hbase/stats/LongColumnStatsAggregator.java | 36 + .../stats/StringColumnStatsAggregator.java | 36 + .../hive/metastore/parser/ExpressionTree.java | 9 +- .../metastore/hbase/hbase_metastore_proto.proto | 282 + .../DummyRawStoreControlledCommit.java | 56 +- .../DummyRawStoreForJdoConnection.java | 50 +- .../hadoop/hive/metastore/TestObjectStore.java | 43 +- .../hadoop/hive/metastore/hbase/MockUtils.java | 199 + .../hbase/TestHBaseAggregateStatsCache.java | 316 + .../hbase/TestHBaseFilterPlanUtil.java | 483 + .../hive/metastore/hbase/TestHBaseStore.java | 1307 + .../metastore/hbase/TestHBaseStoreCached.java | 378 + .../hbase/TestSharedStorageDescriptor.java | 153 + pom.xml | 3 +- .../hadoop/hive/ql/plan/api/Adjacency.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Graph.java | 2 +- .../hadoop/hive/ql/plan/api/Operator.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Query.java | 2 +- .../hadoop/hive/ql/plan/api/QueryPlan.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Stage.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Task.java | 2 +- .../java/org/apache/hadoop/hive/ql/Driver.java | 6 + .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 64 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 9 +- .../hadoop/hive/ql/metadata/Partition.java | 29 +- .../AuthorizationPreEventListener.java | 2 +- .../apache/hadoop/hive/ql/stats/StatsUtils.java | 3 + .../hive/metastore/TestMetastoreExpr.java | 2 +- .../dynpart_sort_opt_vectorization.q | 2 + .../clientpositive/dynpart_sort_optimization.q | 2 + .../dynpart_sort_opt_vectorization.q.out | 12 +- .../dynpart_sort_optimization.q.out | 12 +- .../tez/dynpart_sort_opt_vectorization.q.out | 12 +- .../tez/dynpart_sort_optimization.q.out | 12 +- ql/src/test/templates/TestCliDriver.vm | 3 +- .../hadoop/hive/serde/test/InnerStruct.java | 2 +- .../hadoop/hive/serde/test/ThriftTestObj.java | 2 +- .../hadoop/hive/serde2/thrift/test/Complex.java | 2 +- .../hive/serde2/thrift/test/IntString.java | 2 +- .../hive/serde2/thrift/test/MegaStruct.java | 2 +- .../hive/serde2/thrift/test/MiniStruct.java | 2 +- .../hive/serde2/thrift/test/SetIntString.java | 2 +- .../BinarySortableSerDeWithEndPrefix.java | 41 + .../hadoop/hive/service/HiveClusterStatus.java | 2 +- .../hive/service/HiveServerException.java | 2 +- .../apache/hadoop/hive/service/ThriftHive.java | 2 +- .../service/cli/thrift/TArrayTypeEntry.java | 2 +- .../hive/service/cli/thrift/TBinaryColumn.java | 2 +- .../hive/service/cli/thrift/TBoolColumn.java | 2 +- .../hive/service/cli/thrift/TBoolValue.java | 2 +- .../hive/service/cli/thrift/TByteColumn.java | 2 +- .../hive/service/cli/thrift/TByteValue.java | 2 +- .../hive/service/cli/thrift/TCLIService.java | 2 +- .../cli/thrift/TCancelDelegationTokenReq.java | 2 +- .../cli/thrift/TCancelDelegationTokenResp.java | 2 +- .../service/cli/thrift/TCancelOperationReq.java | 2 +- .../cli/thrift/TCancelOperationResp.java | 2 +- .../service/cli/thrift/TCloseOperationReq.java | 2 +- .../service/cli/thrift/TCloseOperationResp.java | 2 +- .../service/cli/thrift/TCloseSessionReq.java | 2 +- .../service/cli/thrift/TCloseSessionResp.java | 2 +- .../hive/service/cli/thrift/TColumnDesc.java | 2 +- .../hive/service/cli/thrift/TDoubleColumn.java | 2 +- .../hive/service/cli/thrift/TDoubleValue.java | 2 +- .../cli/thrift/TExecuteStatementReq.java | 2 +- .../cli/thrift/TExecuteStatementResp.java | 2 +- .../service/cli/thrift/TFetchResultsReq.java | 2 +- .../service/cli/thrift/TFetchResultsResp.java | 2 +- .../service/cli/thrift/TGetCatalogsReq.java | 2 +- .../service/cli/thrift/TGetCatalogsResp.java | 2 +- .../hive/service/cli/thrift/TGetColumnsReq.java | 2 +- .../service/cli/thrift/TGetColumnsResp.java | 2 +- .../cli/thrift/TGetDelegationTokenReq.java | 2 +- .../cli/thrift/TGetDelegationTokenResp.java | 2 +- .../service/cli/thrift/TGetFunctionsReq.java | 2 +- .../service/cli/thrift/TGetFunctionsResp.java | 2 +- .../hive/service/cli/thrift/TGetInfoReq.java | 2 +- .../hive/service/cli/thrift/TGetInfoResp.java | 2 +- .../cli/thrift/TGetOperationStatusReq.java | 2 +- .../cli/thrift/TGetOperationStatusResp.java | 2 +- .../cli/thrift/TGetResultSetMetadataReq.java | 2 +- .../cli/thrift/TGetResultSetMetadataResp.java | 2 +- .../hive/service/cli/thrift/TGetSchemasReq.java | 2 +- .../service/cli/thrift/TGetSchemasResp.java | 2 +- .../service/cli/thrift/TGetTableTypesReq.java | 2 +- .../service/cli/thrift/TGetTableTypesResp.java | 2 +- .../hive/service/cli/thrift/TGetTablesReq.java | 2 +- .../hive/service/cli/thrift/TGetTablesResp.java | 2 +- .../service/cli/thrift/TGetTypeInfoReq.java | 2 +- .../service/cli/thrift/TGetTypeInfoResp.java | 2 +- .../service/cli/thrift/THandleIdentifier.java | 2 +- .../hive/service/cli/thrift/TI16Column.java | 2 +- .../hive/service/cli/thrift/TI16Value.java | 2 +- .../hive/service/cli/thrift/TI32Column.java | 2 +- .../hive/service/cli/thrift/TI32Value.java | 2 +- .../hive/service/cli/thrift/TI64Column.java | 2 +- .../hive/service/cli/thrift/TI64Value.java | 2 +- .../hive/service/cli/thrift/TMapTypeEntry.java | 2 +- .../service/cli/thrift/TOpenSessionReq.java | 2 +- .../service/cli/thrift/TOpenSessionResp.java | 2 +- .../service/cli/thrift/TOperationHandle.java | 2 +- .../service/cli/thrift/TPrimitiveTypeEntry.java | 2 +- .../cli/thrift/TRenewDelegationTokenReq.java | 2 +- .../cli/thrift/TRenewDelegationTokenResp.java | 2 +- .../apache/hive/service/cli/thrift/TRow.java | 2 +- .../apache/hive/service/cli/thrift/TRowSet.java | 2 +- .../hive/service/cli/thrift/TSessionHandle.java | 2 +- .../apache/hive/service/cli/thrift/TStatus.java | 2 +- .../hive/service/cli/thrift/TStringColumn.java | 2 +- .../hive/service/cli/thrift/TStringValue.java | 2 +- .../service/cli/thrift/TStructTypeEntry.java | 2 +- .../hive/service/cli/thrift/TTableSchema.java | 2 +- .../hive/service/cli/thrift/TTypeDesc.java | 2 +- .../service/cli/thrift/TTypeQualifiers.java | 2 +- .../service/cli/thrift/TUnionTypeEntry.java | 2 +- .../cli/thrift/TUserDefinedTypeEntry.java | 2 +- .../gen-py/hive_service/ThriftHive-remote | 49 +- 299 files changed, 74878 insertions(+), 7084 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/52383033/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/52383033/data/conf/tez/hive-site.xml ---------------------------------------------------------------------- diff --cc data/conf/tez/hive-site.xml index b4abe90,bcda3ea..c4c2a12 --- a/data/conf/tez/hive-site.xml +++ b/data/conf/tez/hive-site.xml @@@ -254,13 -254,13 +254,23 @@@ </property> <property> + <name>hive.tez.java.opts</name> + <value> -Dlog4j.configurationFile=tez-container-log4j2.xml -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA </value> +</property> + +<property> + <name>tez.am.launch.cmd-opts</name> + <value> -Dlog4j.configurationFile=tez-container-log4j2.xml -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA </value> +</property> + ++<property> + <name>hive.metastore.fastpath</name> + <value>true</value> + </property> + + <property> + <name>hive.metastore.rawstore.impl</name> + <value>org.apache.hadoop.hive.metastore.hbase.HBaseStore</value> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/hive/blob/52383033/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/52383033/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/52383033/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/52383033/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/52383033/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/52383033/ql/src/java/org/apache/hadoop/hive/ql/Driver.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/52383033/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/52383033/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/52383033/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/52383033/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/52383033/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/52383033/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/52383033/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out ----------------------------------------------------------------------