HIVE-18755 Modifications to the metastore for catalogs (Alan Gates, reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ba8a99e1 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ba8a99e1 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ba8a99e1 Branch: refs/heads/master Commit: ba8a99e115b6726e0de3ed2ac60b25037156b084 Parents: 5109602 Author: Alan Gates <[email protected]> Authored: Fri Mar 30 18:24:21 2018 -0700 Committer: Alan Gates <[email protected]> Committed: Fri Mar 30 18:24:21 2018 -0700 ---------------------------------------------------------------------- .../cli/SemanticAnalysis/CreateTableHook.java | 2 +- .../apache/hive/hcatalog/cli/TestPermsGrp.java | 4 +- .../listener/DbNotificationListener.java | 19 + .../hcatalog/listener/NotificationListener.java | 2 +- .../listener/DummyRawStoreFailEvent.java | 317 +- .../hive/metastore/TestAcidTableSetup.java | 14 +- .../hive/metastore/TestMetaStoreMetrics.java | 2 +- .../hive/ql/parse/TestReplicationScenarios.java | 13 +- .../upgrade/derby/053-HIVE-18755.derby.sql | 54 + .../derby/upgrade-2.3.0-to-3.0.0.derby.sql | 1 + .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 5 +- .../hive/ql/exec/MaterializedViewTask.java | 6 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 25 +- .../ql/metadata/SessionHiveMetaStoreClient.java | 28 +- .../hive/ql/parse/BaseSemanticAnalyzer.java | 91 +- .../hive/ql/parse/DDLSemanticAnalyzer.java | 20 +- .../hive/ql/parse/MacroSemanticAnalyzer.java | 2 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 7 +- .../HiveAuthorizationProviderBase.java | 4 +- .../AuthorizationMetaStoreFilterHook.java | 3 +- .../hive/ql/txn/compactor/CompactorThread.java | 6 +- .../alter_table_constraint_duplicate_pk.q.out | 2 +- .../clientnegative/create_view_failure2.q.out | 2 +- .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 6501 ++++++++++------ .../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 575 +- .../ThriftHiveMetastore_server.skeleton.cpp | 22 +- .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 7270 ++++++++++------- .../gen/thrift/gen-cpp/hive_metastore_types.h | 700 +- .../hive/metastore/api/AbortTxnsRequest.java | 32 +- .../api/AddCheckConstraintRequest.java | 36 +- .../api/AddDefaultConstraintRequest.java | 36 +- .../metastore/api/AddDynamicPartitions.java | 32 +- .../metastore/api/AddForeignKeyRequest.java | 36 +- .../api/AddNotNullConstraintRequest.java | 36 +- .../metastore/api/AddPartitionsRequest.java | 150 +- .../hive/metastore/api/AddPartitionsResult.java | 36 +- .../metastore/api/AddPrimaryKeyRequest.java | 36 +- .../api/AddUniqueConstraintRequest.java | 36 +- .../hadoop/hive/metastore/api/AggrStats.java | 36 +- .../api/AllocateTableWriteIdsRequest.java | 32 +- .../api/AllocateTableWriteIdsResponse.java | 36 +- .../hadoop/hive/metastore/api/Catalog.java | 606 ++ .../metastore/api/CheckConstraintsRequest.java | 117 +- .../metastore/api/CheckConstraintsResponse.java | 36 +- .../metastore/api/ClearFileMetadataRequest.java | 32 +- .../hive/metastore/api/ClientCapabilities.java | 32 +- .../hive/metastore/api/ColumnStatistics.java | 36 +- .../metastore/api/ColumnStatisticsDesc.java | 114 +- .../hive/metastore/api/CompactionRequest.java | 44 +- .../metastore/api/CreateCatalogRequest.java | 400 + .../hive/metastore/api/CreationMetadata.java | 165 +- .../hadoop/hive/metastore/api/Database.java | 158 +- .../api/DefaultConstraintsRequest.java | 117 +- .../api/DefaultConstraintsResponse.java | 36 +- .../hive/metastore/api/DropCatalogRequest.java | 395 + .../metastore/api/DropConstraintRequest.java | 112 +- .../metastore/api/DropPartitionsRequest.java | 114 +- .../metastore/api/DropPartitionsResult.java | 36 +- .../hive/metastore/api/EnvironmentContext.java | 44 +- .../metastore/api/FindSchemasByColsResp.java | 36 +- .../hive/metastore/api/FireEventRequest.java | 146 +- .../hive/metastore/api/ForeignKeysRequest.java | 113 +- .../hive/metastore/api/ForeignKeysResponse.java | 36 +- .../hadoop/hive/metastore/api/Function.java | 149 +- .../metastore/api/GetAllFunctionsResponse.java | 36 +- .../hive/metastore/api/GetCatalogRequest.java | 395 + .../hive/metastore/api/GetCatalogResponse.java | 400 + .../hive/metastore/api/GetCatalogsResponse.java | 444 ++ .../api/GetFileMetadataByExprRequest.java | 32 +- .../api/GetFileMetadataByExprResult.java | 48 +- .../metastore/api/GetFileMetadataRequest.java | 32 +- .../metastore/api/GetFileMetadataResult.java | 44 +- .../metastore/api/GetOpenTxnsInfoResponse.java | 36 +- .../hive/metastore/api/GetOpenTxnsResponse.java | 32 +- .../hive/metastore/api/GetTableRequest.java | 114 +- .../hive/metastore/api/GetTablesRequest.java | 146 +- .../hive/metastore/api/GetTablesResult.java | 36 +- .../metastore/api/GetValidWriteIdsRequest.java | 32 +- .../metastore/api/GetValidWriteIdsResponse.java | 36 +- .../api/HeartbeatTxnRangeResponse.java | 64 +- .../hive/metastore/api/HiveObjectRef.java | 113 +- .../hadoop/hive/metastore/api/ISchema.java | 178 +- .../hadoop/hive/metastore/api/ISchemaName.java | 130 +- .../metastore/api/InsertEventRequestData.java | 64 +- .../hadoop/hive/metastore/api/LockRequest.java | 36 +- .../hive/metastore/api/Materialization.java | 32 +- .../api/NotNullConstraintsRequest.java | 117 +- .../api/NotNullConstraintsResponse.java | 36 +- .../hive/metastore/api/NotificationEvent.java | 114 +- .../api/NotificationEventResponse.java | 36 +- .../api/NotificationEventsCountRequest.java | 112 +- .../hive/metastore/api/OpenTxnsResponse.java | 32 +- .../hadoop/hive/metastore/api/Partition.java | 190 +- .../api/PartitionListComposingSpec.java | 36 +- .../hive/metastore/api/PartitionSpec.java | 114 +- .../api/PartitionSpecWithSharedSD.java | 36 +- .../metastore/api/PartitionValuesRequest.java | 186 +- .../metastore/api/PartitionValuesResponse.java | 36 +- .../hive/metastore/api/PartitionValuesRow.java | 32 +- .../hive/metastore/api/PartitionWithoutSD.java | 76 +- .../metastore/api/PartitionsByExprRequest.java | 114 +- .../metastore/api/PartitionsByExprResult.java | 36 +- .../metastore/api/PartitionsStatsRequest.java | 176 +- .../metastore/api/PartitionsStatsResult.java | 76 +- .../hive/metastore/api/PrimaryKeysRequest.java | 112 +- .../hive/metastore/api/PrimaryKeysResponse.java | 36 +- .../metastore/api/PutFileMetadataRequest.java | 64 +- .../hive/metastore/api/RequestPartsSpec.java | 68 +- .../hive/metastore/api/SQLCheckConstraint.java | 202 +- .../metastore/api/SQLDefaultConstraint.java | 202 +- .../hive/metastore/api/SQLForeignKey.java | 113 +- .../metastore/api/SQLNotNullConstraint.java | 190 +- .../hive/metastore/api/SQLPrimaryKey.java | 113 +- .../hive/metastore/api/SQLUniqueConstraint.java | 202 +- .../hadoop/hive/metastore/api/Schema.java | 80 +- .../hive/metastore/api/SchemaVersion.java | 36 +- .../hadoop/hive/metastore/api/SerDeInfo.java | 44 +- .../api/SetPartitionsStatsRequest.java | 36 +- .../hive/metastore/api/ShowCompactResponse.java | 36 +- .../hive/metastore/api/ShowLocksResponse.java | 36 +- .../hadoop/hive/metastore/api/SkewedInfo.java | 164 +- .../hive/metastore/api/StorageDescriptor.java | 148 +- .../apache/hadoop/hive/metastore/api/Table.java | 194 +- .../hadoop/hive/metastore/api/TableMeta.java | 114 +- .../hive/metastore/api/TableStatsRequest.java | 144 +- .../hive/metastore/api/TableStatsResult.java | 36 +- .../hive/metastore/api/TableValidWriteIds.java | 32 +- .../hive/metastore/api/ThriftHiveMetastore.java | 7313 ++++++++++++++---- .../metastore/api/UniqueConstraintsRequest.java | 117 +- .../api/UniqueConstraintsResponse.java | 36 +- .../hive/metastore/api/WMFullResourcePlan.java | 144 +- .../api/WMGetAllResourcePlanResponse.java | 36 +- .../WMGetTriggersForResourePlanResponse.java | 36 +- .../api/WMValidateResourcePlanResponse.java | 64 +- .../gen-php/metastore/ThriftHiveMetastore.php | 3016 +++++--- .../src/gen/thrift/gen-php/metastore/Types.php | 3627 ++++++--- .../hive_metastore/ThriftHiveMetastore-remote | 36 +- .../hive_metastore/ThriftHiveMetastore.py | 2106 +++-- .../gen/thrift/gen-py/hive_metastore/ttypes.py | 2793 ++++--- .../gen/thrift/gen-rb/hive_metastore_types.rb | 325 +- .../gen/thrift/gen-rb/thrift_hive_metastore.rb | 280 +- .../hive/metastore/AggregateStatsCache.java | 46 +- .../hadoop/hive/metastore/AlterHandler.java | 29 +- .../DefaultMetaStoreFilterHookImpl.java | 11 +- .../hadoop/hive/metastore/HiveAlterHandler.java | 134 +- .../hadoop/hive/metastore/HiveMetaStore.java | 1253 ++- .../hive/metastore/HiveMetaStoreClient.java | 1304 ++-- .../hadoop/hive/metastore/IHMSHandler.java | 25 +- .../hadoop/hive/metastore/IMetaStoreClient.java | 2119 ++++- .../MaterializationsInvalidationCache.java | 11 +- .../hive/metastore/MetaStoreDirectSql.java | 290 +- .../hive/metastore/MetaStoreEventListener.java | 8 + .../hive/metastore/MetaStoreFilterHook.java | 67 +- .../metastore/MetaStoreListenerNotifier.java | 6 + .../hadoop/hive/metastore/ObjectStore.java | 1805 +++-- .../apache/hadoop/hive/metastore/RawStore.java | 1008 ++- .../hive/metastore/StatObjectConverter.java | 6 + .../TransactionalValidationListener.java | 4 +- .../apache/hadoop/hive/metastore/Warehouse.java | 80 +- .../hadoop/hive/metastore/cache/CacheUtils.java | 40 +- .../hive/metastore/cache/CachedStore.java | 938 ++- .../hive/metastore/cache/SharedCache.java | 380 +- .../client/builder/CatalogBuilder.java | 62 + .../client/builder/ConstraintBuilder.java | 47 +- .../client/builder/DatabaseBuilder.java | 39 +- .../client/builder/FunctionBuilder.java | 37 +- .../client/builder/ISchemaBuilder.java | 12 +- .../client/builder/PartitionBuilder.java | 28 +- .../builder/SQLCheckConstraintBuilder.java | 51 + .../builder/SQLDefaultConstraintBuilder.java | 51 + .../client/builder/SQLForeignKeyBuilder.java | 48 +- .../builder/SQLNotNullConstraintBuilder.java | 23 +- .../client/builder/SQLPrimaryKeyBuilder.java | 18 +- .../builder/SQLUniqueConstraintBuilder.java | 17 +- .../client/builder/SchemaVersionBuilder.java | 14 +- .../metastore/client/builder/TableBuilder.java | 60 +- .../hive/metastore/conf/MetastoreConf.java | 6 + .../metastore/events/CreateCatalogEvent.java | 39 + .../hive/metastore/events/DropCatalogEvent.java | 39 + .../metastore/events/DropConstraintEvent.java | 8 +- .../hive/metastore/events/InsertEvent.java | 6 +- .../metastore/events/PreCreateCatalogEvent.java | 39 + .../metastore/events/PreDropCatalogEvent.java | 39 + .../hive/metastore/events/PreEventContext.java | 5 +- .../events/PreLoadPartitionDoneEvent.java | 8 +- .../metastore/events/PreReadCatalogEvent.java | 39 + .../messaging/CreateCatalogMessage.java | 25 + .../metastore/messaging/DropCatalogMessage.java | 25 + .../hive/metastore/messaging/EventMessage.java | 4 +- .../metastore/messaging/MessageFactory.java | 7 + .../json/JSONCreateCatalogMessage.java | 79 + .../messaging/json/JSONDropCatalogMessage.java | 66 + .../messaging/json/JSONMessageFactory.java | 13 + .../hadoop/hive/metastore/model/MCatalog.java | 58 + .../hive/metastore/model/MCreationMetadata.java | 12 +- .../hadoop/hive/metastore/model/MDatabase.java | 12 +- .../hive/metastore/model/MNotificationLog.java | 12 +- .../model/MPartitionColumnStatistics.java | 9 + .../hive/metastore/model/MPartitionEvent.java | 9 +- .../metastore/model/MTableColumnStatistics.java | 9 + .../spec/CompositePartitionSpecProxy.java | 32 + .../spec/PartitionListComposingSpecProxy.java | 18 + .../partition/spec/PartitionSpecProxy.java | 19 + .../spec/PartitionSpecWithSharedSDProxy.java | 19 +- .../hadoop/hive/metastore/tools/SmokeTest.java | 17 +- .../hive/metastore/utils/MetaStoreUtils.java | 143 +- .../src/main/resources/package.jdo | 39 +- .../main/sql/derby/hive-schema-3.0.0.derby.sql | 103 +- .../sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql | 58 + .../main/sql/mssql/hive-schema-3.0.0.mssql.sql | 27 +- .../sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql | 59 + .../main/sql/mysql/hive-schema-3.0.0.mysql.sql | 20 +- .../sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql | 57 + .../sql/oracle/hive-schema-3.0.0.oracle.sql | 22 +- .../oracle/upgrade-2.3.0-to-3.0.0.oracle.sql | 58 +- .../sql/postgres/hive-schema-3.0.0.postgres.sql | 18 +- .../upgrade-2.3.0-to-3.0.0.postgres.sql | 56 + .../src/main/thrift/hive_metastore.thrift | 202 +- .../DummyRawStoreControlledCommit.java | 308 +- .../DummyRawStoreForJdoConnection.java | 201 +- .../HiveMetaStoreClientPreCatalog.java | 3286 ++++++++ .../InjectableBehaviourObjectStore.java | 8 +- .../hive/metastore/MetaStoreTestUtils.java | 29 + .../hive/metastore/TestAggregateStatsCache.java | 26 +- .../metastore/TestCatalogNonDefaultClient.java | 74 + .../metastore/TestCatalogNonDefaultSvr.java | 70 + .../hive/metastore/TestCatalogOldClient.java | 44 + .../hadoop/hive/metastore/TestFilterHooks.java | 39 +- .../hive/metastore/TestHiveAlterHandler.java | 16 +- .../hive/metastore/TestHiveMetaStore.java | 227 +- .../TestHiveMetaStorePartitionSpecs.java | 9 +- .../TestHiveMetaStoreSchemaMethods.java | 214 +- .../metastore/TestHiveMetaStoreTimeout.java | 21 +- ...TestHiveMetaStoreWithEnvironmentContext.java | 10 +- .../hive/metastore/TestMarkPartition.java | 10 +- .../TestMetaStoreEndFunctionListener.java | 11 +- .../metastore/TestMetaStoreEventListener.java | 21 +- .../TestMetaStoreEventListenerOnlyOnCommit.java | 24 +- ...stMetaStoreMaterializationsCacheCleaner.java | 9 +- .../metastore/TestNonCatCallsWithCatalog.java | 1126 +++ .../hadoop/hive/metastore/TestObjectStore.java | 204 +- .../metastore/TestObjectStoreSchemaMethods.java | 145 +- .../hadoop/hive/metastore/TestOldSchema.java | 29 +- .../TestRemoteHiveMetaStoreIpAddress.java | 7 +- .../hive/metastore/TestRetryingHMSHandler.java | 13 +- .../apache/hadoop/hive/metastore/TestStats.java | 728 ++ .../hive/metastore/VerifyingObjectStore.java | 40 +- .../hive/metastore/cache/TestCachedStore.java | 226 +- .../metastore/cache/TestCatalogCaching.java | 142 + .../client/MetaStoreFactoryForTests.java | 2 + .../metastore/client/TestAddPartitions.java | 120 +- .../client/TestAddPartitionsFromPartSpec.java | 22 +- .../metastore/client/TestAlterPartitions.java | 227 +- .../metastore/client/TestAppendPartitions.java | 90 +- .../hive/metastore/client/TestCatalogs.java | 217 + .../metastore/client/TestCheckConstraint.java | 360 + .../hive/metastore/client/TestDatabases.java | 184 +- .../metastore/client/TestDefaultConstraint.java | 360 + .../metastore/client/TestDropPartitions.java | 96 +- .../client/TestExchangePartitions.java | 13 +- .../hive/metastore/client/TestForeignKey.java | 535 ++ .../hive/metastore/client/TestFunctions.java | 126 +- .../metastore/client/TestGetPartitions.java | 120 +- .../hive/metastore/client/TestGetTableMeta.java | 89 +- .../metastore/client/TestListPartitions.java | 236 +- .../metastore/client/TestNotNullConstraint.java | 352 + .../hive/metastore/client/TestPrimaryKey.java | 465 ++ .../TestTablesCreateDropAlterTruncate.java | 337 +- .../metastore/client/TestTablesGetExists.java | 129 +- .../hive/metastore/client/TestTablesList.java | 73 +- .../metastore/client/TestUniqueConstraint.java | 353 + .../minihms/AbstractMetaStoreService.java | 4 + .../tools/TestSchemaToolForMetastore.java | 31 +- .../src/test/resources/log4j2.properties | 74 +- 274 files changed, 51952 insertions(+), 17293 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java index b6fe9ce..a377805 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java @@ -106,7 +106,7 @@ final class CreateTableHook extends HCatSemanticAnalyzerBase { case HiveParser.TOK_TABLEPARTCOLS: List<FieldSchema> partCols = BaseSemanticAnalyzer - .getColumns(child, false); + .getColumns(child, false, context.getConf()); for (FieldSchema fs : partCols) { if (!fs.getType().equalsIgnoreCase("string")) { throw new SemanticException( http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java index a686579..8a2151c 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java @@ -156,7 +156,7 @@ public class TestPermsGrp extends TestCase { fail(); } catch (Exception e) { assertTrue(e instanceof NoSuchObjectException); - assertEquals("default.simptbl table not found", e.getMessage()); + assertEquals("hive.default.simptbl table not found", e.getMessage()); } // test for invalid group name @@ -177,7 +177,7 @@ public class TestPermsGrp extends TestCase { fail(); } catch (Exception e) { assertTrue(e instanceof NoSuchObjectException); - assertEquals("default.simptbl table not found", e.getMessage()); + assertEquals("hive.default.simptbl table not found", e.getMessage()); } try { // neither dir should get created. http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java index d647181..8523428 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java @@ -74,6 +74,8 @@ import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; + /** * An implementation of {@link org.apache.hadoop.hive.metastore.MetaStoreEventListener} that * stores events in the database. @@ -140,6 +142,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener NotificationEvent event = new NotificationEvent(0, now(), EventType.CREATE_TABLE.toString(), msgFactory .buildCreateTableMessage(t, new FileIterator(t.getSd().getLocation())).toString()); + event.setCatName(t.isSetCatName() ? t.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); process(event, tableEvent); @@ -155,6 +158,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener NotificationEvent event = new NotificationEvent(0, now(), EventType.DROP_TABLE.toString(), msgFactory .buildDropTableMessage(t).toString()); + event.setCatName(t.isSetCatName() ? t.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); process(event, tableEvent); @@ -171,6 +175,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener NotificationEvent event = new NotificationEvent(0, now(), EventType.ALTER_TABLE.toString(), msgFactory .buildAlterTableMessage(before, after, tableEvent.getIsTruncateOp()).toString()); + event.setCatName(after.isSetCatName() ? after.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(after.getDbName()); event.setTableName(after.getTableName()); process(event, tableEvent); @@ -279,6 +284,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener new PartitionFilesIterator(partitionEvent.getPartitionIterator(), t)).toString(); NotificationEvent event = new NotificationEvent(0, now(), EventType.ADD_PARTITION.toString(), msg); + event.setCatName(t.isSetCatName() ? t.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); process(event, partitionEvent); @@ -294,6 +300,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener NotificationEvent event = new NotificationEvent(0, now(), EventType.DROP_PARTITION.toString(), msgFactory .buildDropPartitionMessage(t, partitionEvent.getPartitionIterator()).toString()); + event.setCatName(t.isSetCatName() ? t.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); process(event, partitionEvent); @@ -310,6 +317,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener NotificationEvent event = new NotificationEvent(0, now(), EventType.ALTER_PARTITION.toString(), msgFactory .buildAlterPartitionMessage(partitionEvent.getTable(), before, after, partitionEvent.getIsTruncateOp()).toString()); + event.setCatName(before.isSetCatName() ? before.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(before.getDbName()); event.setTableName(before.getTableName()); process(event, partitionEvent); @@ -325,6 +333,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener NotificationEvent event = new NotificationEvent(0, now(), EventType.CREATE_DATABASE.toString(), msgFactory .buildCreateDatabaseMessage(db).toString()); + event.setCatName(db.isSetCatalogName() ? db.getCatalogName() : DEFAULT_CATALOG_NAME); event.setDbName(db.getName()); process(event, dbEvent); } @@ -339,6 +348,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener NotificationEvent event = new NotificationEvent(0, now(), EventType.DROP_DATABASE.toString(), msgFactory .buildDropDatabaseMessage(db).toString()); + event.setCatName(db.isSetCatalogName() ? db.getCatalogName() : DEFAULT_CATALOG_NAME); event.setDbName(db.getName()); process(event, dbEvent); } @@ -354,6 +364,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener NotificationEvent event = new NotificationEvent(0, now(), EventType.ALTER_DATABASE.toString(), msgFactory .buildAlterDatabaseMessage(oldDb, newDb).toString()); + event.setCatName(oldDb.isSetCatalogName() ? oldDb.getCatalogName() : DEFAULT_CATALOG_NAME); event.setDbName(oldDb.getName()); process(event, dbEvent); } @@ -368,6 +379,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener NotificationEvent event = new NotificationEvent(0, now(), EventType.CREATE_FUNCTION.toString(), msgFactory .buildCreateFunctionMessage(fn).toString()); + event.setCatName(fn.isSetCatName() ? fn.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(fn.getDbName()); process(event, fnEvent); } @@ -382,6 +394,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener NotificationEvent event = new NotificationEvent(0, now(), EventType.DROP_FUNCTION.toString(), msgFactory .buildDropFunctionMessage(fn).toString()); + event.setCatName(fn.isSetCatName() ? fn.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(fn.getDbName()); process(event, fnEvent); } @@ -419,6 +432,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener insertEvent.getPartitionObj(), insertEvent.isReplace(), new FileChksumIterator(insertEvent.getFiles(), insertEvent.getFileChecksums())) .toString()); + event.setCatName(tableObj.isSetCatName() ? tableObj.getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(tableObj.getDbName()); event.setTableName(tableObj.getTableName()); process(event, insertEvent); @@ -445,6 +459,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener NotificationEvent event = new NotificationEvent(0, now(), EventType.ADD_PRIMARYKEY.toString(), msgFactory .buildAddPrimaryKeyMessage(addPrimaryKeyEvent.getPrimaryKeyCols()).toString()); + event.setCatName(cols.get(0).isSetCatName() ? cols.get(0).getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(cols.get(0).getTable_db()); event.setTableName(cols.get(0).getTable_name()); process(event, addPrimaryKeyEvent); @@ -462,6 +477,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener NotificationEvent event = new NotificationEvent(0, now(), EventType.ADD_FOREIGNKEY.toString(), msgFactory .buildAddForeignKeyMessage(addForeignKeyEvent.getForeignKeyCols()).toString()); + event.setCatName(cols.get(0).isSetCatName() ? cols.get(0).getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(cols.get(0).getPktable_db()); event.setTableName(cols.get(0).getPktable_name()); process(event, addForeignKeyEvent); @@ -479,6 +495,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener NotificationEvent event = new NotificationEvent(0, now(), EventType.ADD_UNIQUECONSTRAINT.toString(), msgFactory .buildAddUniqueConstraintMessage(addUniqueConstraintEvent.getUniqueConstraintCols()).toString()); + event.setCatName(cols.get(0).isSetCatName() ? cols.get(0).getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(cols.get(0).getTable_db()); event.setTableName(cols.get(0).getTable_name()); process(event, addUniqueConstraintEvent); @@ -496,6 +513,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener NotificationEvent event = new NotificationEvent(0, now(), EventType.ADD_NOTNULLCONSTRAINT.toString(), msgFactory .buildAddNotNullConstraintMessage(addNotNullConstraintEvent.getNotNullConstraintCols()).toString()); + event.setCatName(cols.get(0).isSetCatName() ? cols.get(0).getCatName() : DEFAULT_CATALOG_NAME); event.setDbName(cols.get(0).getTable_db()); event.setTableName(cols.get(0).getTable_name()); process(event, addNotNullConstraintEvent); @@ -514,6 +532,7 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener NotificationEvent event = new NotificationEvent(0, now(), EventType.DROP_CONSTRAINT.toString(), msgFactory .buildDropConstraintMessage(dbName, tableName, constraintName).toString()); + event.setCatName(dropConstraintEvent.getCatName()); event.setDbName(dbName); event.setTableName(tableName); process(event, dropConstraintEvent); http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java ---------------------------------------------------------------------- diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java index 53246a0..649d901 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java @@ -207,7 +207,7 @@ public class NotificationListener extends MetaStoreEventListener { Configuration conf = handler.getConf(); Table newTbl; try { - newTbl = handler.get_table_core(tbl.getDbName(), tbl.getTableName()) + newTbl = handler.get_table_core(tbl.getCatName(), tbl.getDbName(), tbl.getTableName()) .deepCopy(); newTbl.getParameters().put( HCatConstants.HCAT_MSGBUS_TOPIC_NAME, http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java ---------------------------------------------------------------------- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index 5cc407a..4697f60 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -20,6 +20,7 @@ package org.apache.hive.hcatalog.listener; import org.apache.hadoop.hive.metastore.api.ISchemaName; import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; +import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -146,6 +147,40 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override + public void createCatalog(Catalog cat) throws MetaException { + if (shouldEventSucceed) { + objectStore.createCatalog(cat); + } else { + throw new RuntimeException("Failed event"); + } + } + + @Override + public void alterCatalog(String catName, Catalog cat) throws MetaException, + InvalidOperationException { + objectStore.alterCatalog(catName, cat); + } + + @Override + public Catalog getCatalog(String catalogName) throws NoSuchObjectException, MetaException { + return objectStore.getCatalog(catalogName); + } + + @Override + public List<String> getCatalogs() throws MetaException { + return objectStore.getCatalogs(); + } + + @Override + public void dropCatalog(String catalogName) throws NoSuchObjectException, MetaException { + if (shouldEventSucceed) { + objectStore.dropCatalog(catalogName); + } else { + throw new RuntimeException("Event failed."); + } + } + + @Override public void createDatabase(Database db) throws InvalidObjectException, MetaException { if (shouldEventSucceed) { objectStore.createDatabase(db); @@ -155,34 +190,34 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override - public Database getDatabase(String dbName) throws NoSuchObjectException { - return objectStore.getDatabase(dbName); + public Database getDatabase(String catName, String dbName) throws NoSuchObjectException { + return objectStore.getDatabase(catName, dbName); } @Override - public boolean dropDatabase(String dbName) + public boolean dropDatabase(String catName, String dbName) throws NoSuchObjectException, MetaException { if (shouldEventSucceed) { - return objectStore.dropDatabase(dbName); + return objectStore.dropDatabase(catName, dbName); } else { throw new RuntimeException("Event failed."); } } @Override - public boolean alterDatabase(String dbName, Database db) + public boolean alterDatabase(String catName, String dbName, Database db) throws NoSuchObjectException, MetaException { - return objectStore.alterDatabase(dbName, db); + return objectStore.alterDatabase(catName, dbName, db); } @Override - public List<String> getDatabases(String pattern) throws MetaException { - return objectStore.getDatabases(pattern); + public List<String> getDatabases(String catName, String pattern) throws MetaException { + return objectStore.getDatabases(catName, pattern); } @Override - public List<String> getAllDatabases() throws MetaException { - return objectStore.getAllDatabases(); + public List<String> getAllDatabases(String catName) throws MetaException { + return objectStore.getAllDatabases(catName); } @Override @@ -210,19 +245,19 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override - public boolean dropTable(String dbName, String tableName) + public boolean dropTable(String catName, String dbName, String tableName) throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { if (shouldEventSucceed) { - return objectStore.dropTable(dbName, tableName); + return objectStore.dropTable(catName, dbName, tableName); } else { throw new RuntimeException("Event failed."); } } @Override - public Table getTable(String dbName, String tableName) throws MetaException { - return objectStore.getTable(dbName, tableName); + public Table getTable(String catName, String dbName, String tableName) throws MetaException { + return objectStore.getTable(catName, dbName, tableName); } @Override @@ -232,162 +267,159 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override - public Partition getPartition(String dbName, String tableName, List<String> partVals) + public Partition getPartition(String catName, String dbName, String tableName, List<String> partVals) throws MetaException, NoSuchObjectException { - return objectStore.getPartition(dbName, tableName, partVals); + return objectStore.getPartition(catName, dbName, tableName, partVals); } @Override - public boolean dropPartition(String dbName, String tableName, List<String> partVals) + public boolean dropPartition(String catName, String dbName, String tableName, List<String> partVals) throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { if (shouldEventSucceed) { - return objectStore.dropPartition(dbName, tableName, partVals); + return objectStore.dropPartition(catName, dbName, tableName, partVals); } else { throw new RuntimeException("Event failed."); } } @Override - public List<Partition> getPartitions(String dbName, String tableName, int max) + public List<Partition> getPartitions(String catName, String dbName, String tableName, int max) throws MetaException, NoSuchObjectException { - return objectStore.getPartitions(dbName, tableName, max); + return objectStore.getPartitions(catName, dbName, tableName, max); } @Override - public void updateCreationMetadata(String dbname, String tablename, CreationMetadata cm) + public void updateCreationMetadata(String catName, String dbname, String tablename, CreationMetadata cm) throws MetaException { - objectStore.updateCreationMetadata(dbname, tablename, cm); + objectStore.updateCreationMetadata(catName, dbname, tablename, cm); } - @Override - public void alterTable(String dbName, String name, Table newTable) + public void alterTable(String catName, String dbName, String name, Table newTable) throws InvalidObjectException, MetaException { if (shouldEventSucceed) { - objectStore.alterTable(dbName, name, newTable); + objectStore.alterTable(catName, dbName, name, newTable); } else { throw new RuntimeException("Event failed."); } } @Override - public List<String> getTables(String dbName, String pattern) throws MetaException { - return objectStore.getTables(dbName, pattern); + public List<String> getTables(String catName, String dbName, String pattern) throws MetaException { + return objectStore.getTables(catName, dbName, pattern); } @Override - public List<String> getTables(String dbName, String pattern, TableType tableType) throws MetaException { - return objectStore.getTables(dbName, pattern, tableType); + public List<String> getTables(String catName, String dbName, String pattern, TableType tableType) throws MetaException { + return objectStore.getTables(catName, dbName, pattern, tableType); } @Override - public List<String> getMaterializedViewsForRewriting(String dbName) + public List<String> getMaterializedViewsForRewriting(String catName, String dbName) throws MetaException, NoSuchObjectException { - return objectStore.getMaterializedViewsForRewriting(dbName); + return objectStore.getMaterializedViewsForRewriting(catName, dbName); } @Override - public List<TableMeta> getTableMeta(String dbNames, String tableNames, List<String> tableTypes) + public List<TableMeta> getTableMeta(String catName, String dbNames, String tableNames, List<String> tableTypes) throws MetaException { - return objectStore.getTableMeta(dbNames, tableNames, tableTypes); + return objectStore.getTableMeta(catName, dbNames, tableNames, tableTypes); } @Override - public List<Table> getTableObjectsByName(String dbName, List<String> tableNames) + public List<Table> getTableObjectsByName(String catName, String dbName, List<String> tableNames) throws MetaException, UnknownDBException { - return objectStore.getTableObjectsByName(dbName, tableNames); + return objectStore.getTableObjectsByName(catName, dbName, tableNames); } @Override - public List<String> getAllTables(String dbName) throws MetaException { - return objectStore.getAllTables(dbName); + public List<String> getAllTables(String catName, String dbName) throws MetaException { + return objectStore.getAllTables(catName, dbName); } @Override - public List<String> listTableNamesByFilter(String dbName, String filter, + public List<String> listTableNamesByFilter(String catName, String dbName, String filter, short maxTables) throws MetaException, UnknownDBException { - return objectStore.listTableNamesByFilter(dbName, filter, maxTables); + return objectStore.listTableNamesByFilter(catName, dbName, filter, maxTables); } @Override - public List<String> listPartitionNames(String dbName, String tblName, short maxParts) + public List<String> listPartitionNames(String catName, String dbName, String tblName, short maxParts) throws MetaException { - return objectStore.listPartitionNames(dbName, tblName, maxParts); + return objectStore.listPartitionNames(catName, dbName, tblName, maxParts); } @Override - public PartitionValuesResponse listPartitionValues(String db_name, String tbl_name, List<FieldSchema> cols, boolean applyDistinct, String filter, boolean ascending, List<FieldSchema> order, long maxParts) throws MetaException { + public PartitionValuesResponse listPartitionValues(String catName, String db_name, + String tbl_name, List<FieldSchema> cols, + boolean applyDistinct, String filter, + boolean ascending, List<FieldSchema> order, + long maxParts) throws MetaException { return null; } @Override - public List<String> listPartitionNamesByFilter(String dbName, String tblName, - String filter, short maxParts) throws MetaException { - return objectStore.listPartitionNamesByFilter(dbName, tblName, filter, maxParts); - } - - @Override - public void alterPartition(String dbName, String tblName, List<String> partVals, + public void alterPartition(String catName, String dbName, String tblName, List<String> partVals, Partition newPart) throws InvalidObjectException, MetaException { if (shouldEventSucceed) { - objectStore.alterPartition(dbName, tblName, partVals, newPart); + objectStore.alterPartition(catName, dbName, tblName, partVals, newPart); } else { throw new RuntimeException("Event failed."); } } @Override - public void alterPartitions(String dbName, String tblName, + public void alterPartitions(String catName, String dbName, String tblName, List<List<String>> partValsList, List<Partition> newParts) throws InvalidObjectException, MetaException { - objectStore.alterPartitions(dbName, tblName, partValsList, newParts); + objectStore.alterPartitions(catName, dbName, tblName, partValsList, newParts); } @Override - public List<Partition> getPartitionsByFilter(String dbName, String tblName, + public List<Partition> getPartitionsByFilter(String catName, String dbName, String tblName, String filter, short maxParts) throws MetaException, NoSuchObjectException { - return objectStore.getPartitionsByFilter(dbName, tblName, filter, maxParts); + return objectStore.getPartitionsByFilter(catName, dbName, tblName, filter, maxParts); } @Override - public int getNumPartitionsByFilter(String dbName, String tblName, + public int getNumPartitionsByFilter(String catName, String dbName, String tblName, String filter) throws MetaException, NoSuchObjectException { - return objectStore.getNumPartitionsByFilter(dbName, tblName, filter); + return objectStore.getNumPartitionsByFilter(catName, dbName, tblName, filter); } @Override - public int getNumPartitionsByExpr(String dbName, String tblName, + public int getNumPartitionsByExpr(String catName, String dbName, String tblName, byte[] expr) throws MetaException, NoSuchObjectException { - return objectStore.getNumPartitionsByExpr(dbName, tblName, expr); + return objectStore.getNumPartitionsByExpr(catName, dbName, tblName, expr); } @Override - public List<Partition> getPartitionsByNames(String dbName, String tblName, + public List<Partition> getPartitionsByNames(String catName, String dbName, String tblName, List<String> partNames) throws MetaException, NoSuchObjectException { - return objectStore.getPartitionsByNames(dbName, tblName, partNames); + return objectStore.getPartitionsByNames(catName, dbName, tblName, partNames); } @Override - public boolean getPartitionsByExpr(String dbName, String tblName, byte[] expr, + public boolean getPartitionsByExpr(String catName, String dbName, String tblName, byte[] expr, String defaultPartitionName, short maxParts, List<Partition> result) throws TException { - return objectStore.getPartitionsByExpr( + return objectStore.getPartitionsByExpr(catName, dbName, tblName, expr, defaultPartitionName, maxParts, result); } @Override - public Table markPartitionForEvent(String dbName, String tblName, + public Table markPartitionForEvent(String catName, String dbName, String tblName, Map<String, String> partVals, PartitionEventType evtType) throws MetaException, UnknownTableException, InvalidPartitionException, UnknownPartitionException { - return objectStore.markPartitionForEvent(dbName, tblName, partVals, evtType); + return objectStore.markPartitionForEvent(catName, dbName, tblName, partVals, evtType); } @Override - public boolean isPartitionMarkedForEvent(String dbName, String tblName, + public boolean isPartitionMarkedForEvent(String catName, String dbName, String tblName, Map<String, String> partName, PartitionEventType evtType) throws MetaException, UnknownTableException, InvalidPartitionException, UnknownPartitionException { - return objectStore.isPartitionMarkedForEvent(dbName, tblName, partName, evtType); + return objectStore.isPartitionMarkedForEvent(catName, dbName, tblName, partName, evtType); } @Override @@ -423,32 +455,32 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override - public PrincipalPrivilegeSet getDBPrivilegeSet(String dbName, String userName, + public PrincipalPrivilegeSet getDBPrivilegeSet(String catName, String dbName, String userName, List<String> groupNames) throws InvalidObjectException, MetaException { - return objectStore.getDBPrivilegeSet(dbName, userName, groupNames); + return objectStore.getDBPrivilegeSet(catName, dbName, userName, groupNames); } @Override - public PrincipalPrivilegeSet getTablePrivilegeSet(String dbName, String tableName, + public PrincipalPrivilegeSet getTablePrivilegeSet(String catName, String dbName, String tableName, String userName, List<String> groupNames) throws InvalidObjectException, MetaException { - return objectStore.getTablePrivilegeSet(dbName, tableName, userName, groupNames); + return objectStore.getTablePrivilegeSet(catName, dbName, tableName, userName, groupNames); } @Override - public PrincipalPrivilegeSet getPartitionPrivilegeSet(String dbName, String tableName, + public PrincipalPrivilegeSet getPartitionPrivilegeSet(String catName, String dbName, String tableName, String partition, String userName, List<String> groupNames) throws InvalidObjectException, MetaException { - return objectStore.getPartitionPrivilegeSet(dbName, tableName, partition, + return objectStore.getPartitionPrivilegeSet(catName, dbName, tableName, partition, userName, groupNames); } @Override - public PrincipalPrivilegeSet getColumnPrivilegeSet(String dbName, String tableName, + public PrincipalPrivilegeSet getColumnPrivilegeSet(String catName, String dbName, String tableName, String partitionName, String columnName, String userName, List<String> groupNames) throws InvalidObjectException, MetaException { - return objectStore.getColumnPrivilegeSet(dbName, tableName, partitionName, + return objectStore.getColumnPrivilegeSet(catName, dbName, tableName, partitionName, columnName, userName, groupNames); } @@ -460,40 +492,40 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { @Override public List<HiveObjectPrivilege> listPrincipalDBGrants(String principalName, - PrincipalType principalType, String dbName) { - return objectStore.listPrincipalDBGrants(principalName, principalType, dbName); + PrincipalType principalType, String catName, String dbName) { + return objectStore.listPrincipalDBGrants(principalName, principalType, catName, dbName); } @Override public List<HiveObjectPrivilege> listAllTableGrants(String principalName, - PrincipalType principalType, String dbName, String tableName) { + PrincipalType principalType, String catName, String dbName, String tableName) { return objectStore.listAllTableGrants(principalName, principalType, - dbName, tableName); + catName, dbName, tableName); } @Override public List<HiveObjectPrivilege> listPrincipalPartitionGrants(String principalName, - PrincipalType principalType, String dbName, String tableName, + PrincipalType principalType, String catName, String dbName, String tableName, List<String> partValues, String partName) { return objectStore.listPrincipalPartitionGrants(principalName, principalType, - dbName, tableName, partValues, partName); + catName, dbName, tableName, partValues, partName); } @Override public List<HiveObjectPrivilege> listPrincipalTableColumnGrants(String principalName, - PrincipalType principalType, String dbName, + PrincipalType principalType, String catName, String dbName, String tableName, String columnName) { return objectStore.listPrincipalTableColumnGrants(principalName, principalType, - dbName, tableName, columnName); + catName, dbName, tableName, columnName); } @Override public List<HiveObjectPrivilege> listPrincipalPartitionColumnGrants( - String principalName, PrincipalType principalType, String dbName, String tableName, + String principalName, PrincipalType principalType, String catName, String dbName, String tableName, List<String> partVals, String partName, String columnName) { return objectStore.listPrincipalPartitionColumnGrants(principalName, principalType, - dbName, tableName, partVals, partName, columnName); + catName, dbName, tableName, partVals, partName, columnName); } @Override @@ -535,34 +567,34 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override - public Partition getPartitionWithAuth(String dbName, String tblName, + public Partition getPartitionWithAuth(String catName, String dbName, String tblName, List<String> partVals, String userName, List<String> groupNames) throws MetaException, NoSuchObjectException, InvalidObjectException { - return objectStore.getPartitionWithAuth(dbName, tblName, partVals, userName, + return objectStore.getPartitionWithAuth(catName, dbName, tblName, partVals, userName, groupNames); } @Override - public List<Partition> getPartitionsWithAuth(String dbName, String tblName, + public List<Partition> getPartitionsWithAuth(String catName, String dbName, String tblName, short maxParts, String userName, List<String> groupNames) throws MetaException, NoSuchObjectException, InvalidObjectException { - return objectStore.getPartitionsWithAuth(dbName, tblName, maxParts, userName, + return objectStore.getPartitionsWithAuth(catName, dbName, tblName, maxParts, userName, groupNames); } @Override - public List<String> listPartitionNamesPs(String dbName, String tblName, + public List<String> listPartitionNamesPs(String catName, String dbName, String tblName, List<String> partVals, short maxParts) throws MetaException, NoSuchObjectException { - return objectStore.listPartitionNamesPs(dbName, tblName, partVals, maxParts); + return objectStore.listPartitionNamesPs(catName, dbName, tblName, partVals, maxParts); } @Override - public List<Partition> listPartitionsPsWithAuth(String dbName, String tblName, + public List<Partition> listPartitionsPsWithAuth(String catName, String dbName, String tblName, List<String> partVals, short maxParts, String userName, List<String> groupNames) throws MetaException, InvalidObjectException, NoSuchObjectException { - return objectStore.listPartitionsPsWithAuth(dbName, tblName, partVals, maxParts, + return objectStore.listPartitionsPsWithAuth(catName, dbName, tblName, partVals, maxParts, userName, groupNames); } @@ -607,53 +639,52 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override - public List<HiveObjectPrivilege> listDBGrantsAll(String dbName) { - return objectStore.listDBGrantsAll(dbName); + public List<HiveObjectPrivilege> listDBGrantsAll(String catName, String dbName) { + return objectStore.listDBGrantsAll(catName, dbName); } @Override - public List<HiveObjectPrivilege> listPartitionColumnGrantsAll(String dbName, String tableName, + public List<HiveObjectPrivilege> listPartitionColumnGrantsAll(String catName, String dbName, String tableName, String partitionName, String columnName) { - return objectStore.listPartitionColumnGrantsAll(dbName, tableName, partitionName, columnName); + return objectStore.listPartitionColumnGrantsAll(catName, dbName, tableName, partitionName, columnName); } @Override - public List<HiveObjectPrivilege> listTableGrantsAll(String dbName, String tableName) { - return objectStore.listTableGrantsAll(dbName, tableName); + public List<HiveObjectPrivilege> listTableGrantsAll(String catName, String dbName, String tableName) { + return objectStore.listTableGrantsAll(catName, dbName, tableName); } @Override - public List<HiveObjectPrivilege> listPartitionGrantsAll(String dbName, String tableName, + public List<HiveObjectPrivilege> listPartitionGrantsAll(String catName, String dbName, String tableName, String partitionName) { - return objectStore.listPartitionGrantsAll(dbName, tableName, partitionName); + return objectStore.listPartitionGrantsAll(catName, dbName, tableName, partitionName); } @Override - public List<HiveObjectPrivilege> listTableColumnGrantsAll(String dbName, String tableName, + public List<HiveObjectPrivilege> listTableColumnGrantsAll(String catName, String dbName, String tableName, String columnName) { - return objectStore.listTableColumnGrantsAll(dbName, tableName, columnName); + return objectStore.listTableColumnGrantsAll(catName, dbName, tableName, columnName); } @Override - public ColumnStatistics getTableColumnStatistics(String dbName, String tableName, + public ColumnStatistics getTableColumnStatistics(String catName, String dbName, String tableName, List<String> colNames) throws MetaException, NoSuchObjectException { - return objectStore.getTableColumnStatistics(dbName, tableName, colNames); + return objectStore.getTableColumnStatistics(catName, dbName, tableName, colNames); } @Override - public boolean deleteTableColumnStatistics(String dbName, String tableName, + public boolean deleteTableColumnStatistics(String catName, String dbName, String tableName, String colName) - throws NoSuchObjectException, MetaException, InvalidObjectException, - InvalidInputException { - return objectStore.deleteTableColumnStatistics(dbName, tableName, colName); + throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { + return objectStore.deleteTableColumnStatistics(catName, dbName, tableName, colName); } @Override - public boolean deletePartitionColumnStatistics(String dbName, String tableName, + public boolean deletePartitionColumnStatistics(String catName, String dbName, String tableName, String partName, List<String> partVals, String colName) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { - return objectStore.deletePartitionColumnStatistics(dbName, tableName, partName, + return objectStore.deletePartitionColumnStatistics(catName, dbName, tableName, partName, partVals, colName); } @@ -689,7 +720,7 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { @Override public List<String> getAllTokenIdentifiers() { - return new ArrayList<String>(); + return new ArrayList<>(); } @Override @@ -727,35 +758,35 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override - public List<ColumnStatistics> getPartitionColumnStatistics(String dbName, + public List<ColumnStatistics> getPartitionColumnStatistics(String catName, String dbName, String tblName, List<String> colNames, List<String> partNames) throws MetaException, NoSuchObjectException { - return objectStore.getPartitionColumnStatistics(dbName, tblName , colNames, partNames); + return objectStore.getPartitionColumnStatistics(catName, dbName, tblName , colNames, partNames); } @Override - public boolean doesPartitionExist(String dbName, String tableName, + public boolean doesPartitionExist(String catName, String dbName, String tableName, List<String> partVals) throws MetaException, NoSuchObjectException { - return objectStore.doesPartitionExist(dbName, tableName, partVals); + return objectStore.doesPartitionExist(catName, dbName, tableName, partVals); } @Override - public boolean addPartitions(String dbName, String tblName, List<Partition> parts) + public boolean addPartitions(String catName, String dbName, String tblName, List<Partition> parts) throws InvalidObjectException, MetaException { - return objectStore.addPartitions(dbName, tblName, parts); + return objectStore.addPartitions(catName, dbName, tblName, parts); } @Override - public boolean addPartitions(String dbName, String tblName, PartitionSpecProxy partitionSpec, + public boolean addPartitions(String catName, String dbName, String tblName, PartitionSpecProxy partitionSpec, boolean ifNotExists) throws InvalidObjectException, MetaException { return false; } @Override - public void dropPartitions(String dbName, String tblName, List<String> partNames) + public void dropPartitions(String catName, String dbName, String tblName, List<String> partNames) throws MetaException, NoSuchObjectException { - objectStore.dropPartitions(dbName, tblName, partNames); + objectStore.dropPartitions(catName, dbName, tblName, partNames); } @Override @@ -769,42 +800,42 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override - public void alterFunction(String dbName, String funcName, Function newFunction) + public void alterFunction(String catName, String dbName, String funcName, Function newFunction) throws InvalidObjectException, MetaException { - objectStore.alterFunction(dbName, funcName, newFunction); + objectStore.alterFunction(catName, dbName, funcName, newFunction); } @Override - public void dropFunction(String dbName, String funcName) + public void dropFunction(String catName, String dbName, String funcName) throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { if (shouldEventSucceed) { - objectStore.dropFunction(dbName, funcName); + objectStore.dropFunction(catName, dbName, funcName); } else { throw new RuntimeException("Event failed."); } } @Override - public Function getFunction(String dbName, String funcName) + public Function getFunction(String catName, String dbName, String funcName) throws MetaException { - return objectStore.getFunction(dbName, funcName); + return objectStore.getFunction(catName, dbName, funcName); } @Override - public List<Function> getAllFunctions() + public List<Function> getAllFunctions(String catName) throws MetaException { return Collections.emptyList(); } @Override - public List<String> getFunctions(String dbName, String pattern) + public List<String> getFunctions(String catName, String dbName, String pattern) throws MetaException { - return objectStore.getFunctions(dbName, pattern); + return objectStore.getFunctions(catName, dbName, pattern); } @Override - public AggrStats get_aggr_stats_for(String dbName, + public AggrStats get_aggr_stats_for(String catName, String dbName, String tblName, List<String> partNames, List<String> colNames) throws MetaException { return null; @@ -882,38 +913,38 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override - public List<SQLPrimaryKey> getPrimaryKeys(String db_name, String tbl_name) + public List<SQLPrimaryKey> getPrimaryKeys(String catName, String db_name, String tbl_name) throws MetaException { return null; } @Override - public List<SQLForeignKey> getForeignKeys(String parent_db_name, + public List<SQLForeignKey> getForeignKeys(String catName, String parent_db_name, String parent_tbl_name, String foreign_db_name, String foreign_tbl_name) throws MetaException { return null; } @Override - public List<SQLUniqueConstraint> getUniqueConstraints(String db_name, String tbl_name) + public List<SQLUniqueConstraint> getUniqueConstraints(String catName, String db_name, String tbl_name) throws MetaException { return null; } @Override - public List<SQLNotNullConstraint> getNotNullConstraints(String db_name, String tbl_name) + public List<SQLNotNullConstraint> getNotNullConstraints(String catName, String db_name, String tbl_name) throws MetaException { return null; } @Override - public List<SQLCheckConstraint> getCheckConstraints(String db_name, String tbl_name) + public List<SQLCheckConstraint> getCheckConstraints(String catName, String db_name, String tbl_name) throws MetaException { return null; } @Override - public List<SQLDefaultConstraint> getDefaultConstraints(String db_name, String tbl_name) + public List<SQLDefaultConstraint> getDefaultConstraints(String catName, String db_name, String tbl_name) throws MetaException { return null; } @@ -930,8 +961,9 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override - public void dropConstraint(String dbName, String tableName, - String constraintName) throws NoSuchObjectException { + public void dropConstraint(String catName, String dbName, String tableName, + String constraintName, boolean missingOk) + throws NoSuchObjectException { } @Override @@ -982,7 +1014,7 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override - public WMFullResourcePlan getResourcePlan(String name) throws NoSuchObjectException { + public WMFullResourcePlan getResourcePlan(String name) throws NoSuchObjectException, MetaException { return objectStore.getResourcePlan(name); } @@ -1083,6 +1115,13 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { objectStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); } + @Override + public List<ColStatsObjWithSourceInfo> getPartitionColStatsForDatabase(String catName, String dbName) + throws MetaException, NoSuchObjectException { + // TODO Auto-generated method stub + return null; + } + public void createISchema(ISchema schema) throws AlreadyExistsException, MetaException, NoSuchObjectException { objectStore.createISchema(schema); http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAcidTableSetup.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAcidTableSetup.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAcidTableSetup.java index 62bd94a..505b3c0 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAcidTableSetup.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAcidTableSetup.java @@ -85,7 +85,7 @@ public class TestAcidTableSetup { .setTableName(tblName) .setTableParams(params) .setCols(type.getFields()) - .build(); + .build(conf); client.createTable(t); fail("Expected exception"); } catch (MetaException e) { @@ -102,7 +102,7 @@ public class TestAcidTableSetup { .setTableName(tblName) .setTableParams(params) .setCols(type.getFields()) - .build(); + .build(conf); client.createTable(t); fail("Expected exception"); } catch (MetaException e) { @@ -119,7 +119,7 @@ public class TestAcidTableSetup { .setTableName(tblName) .setTableParams(params) .setCols(type.getFields()) - .build(); + .build(conf); client.createTable(t); fail("Expected exception"); } catch (MetaException e) { @@ -139,7 +139,7 @@ public class TestAcidTableSetup { .setTableParams(params) .setCols(type.getFields()) .setBucketCols(bucketCols) - .build(); + .build(conf); client.createTable(t); fail("Expected exception"); } catch (MetaException e) { @@ -158,7 +158,7 @@ public class TestAcidTableSetup { .setBucketCols(bucketCols) .setInputFormat("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat") .setOutputFormat("org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat") - .build(); + .build(conf); client.createTable(t); assertTrue("CREATE TABLE should succeed", "true".equals(t.getParameters().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL))); @@ -188,7 +188,7 @@ public class TestAcidTableSetup { .setTableName(tblName) .setCols(type.getFields()) .setInputFormat("org.apache.hadoop.mapred.FileInputFormat") - .build(); + .build(conf); client.createTable(t); params.put("transactional", "true"); t.setParameters(params); @@ -210,7 +210,7 @@ public class TestAcidTableSetup { .setBucketCols(bucketCols) .setInputFormat("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat") .setOutputFormat("org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat") - .build(); + .build(conf); client.createTable(t); params.put("transactional", "true"); t.setParameters(params); http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java index 368d517..3f7135c 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java @@ -62,7 +62,7 @@ public class TestMetaStoreMetrics { driver.run("show databases"); //one call by init, one called here. - Assert.assertEquals(2, Metrics.getRegistry().getTimers().get("api_get_all_databases").getCount()); + Assert.assertEquals(2, Metrics.getRegistry().getTimers().get("api_get_databases").getCount()); } @Test http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index 0940938..7f1b662 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -82,6 +82,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals; @@ -3075,11 +3076,11 @@ public class TestReplicationScenarios { try { List<SQLPrimaryKey> pks = metaStoreClientMirror.getPrimaryKeys(new PrimaryKeysRequest(dbName+ "_dupe" , "tbl1")); assertEquals(pks.size(), 2); - List<SQLUniqueConstraint> uks = metaStoreClientMirror.getUniqueConstraints(new UniqueConstraintsRequest(dbName+ "_dupe" , "tbl3")); + List<SQLUniqueConstraint> uks = metaStoreClientMirror.getUniqueConstraints(new UniqueConstraintsRequest(DEFAULT_CATALOG_NAME, dbName+ "_dupe" , "tbl3")); assertEquals(uks.size(), 1); List<SQLForeignKey> fks = metaStoreClientMirror.getForeignKeys(new ForeignKeysRequest(null, null, dbName+ "_dupe" , "tbl2")); assertEquals(fks.size(), 2); - List<SQLNotNullConstraint> nns = metaStoreClientMirror.getNotNullConstraints(new NotNullConstraintsRequest(dbName+ "_dupe" , "tbl3")); + List<SQLNotNullConstraint> nns = metaStoreClientMirror.getNotNullConstraints(new NotNullConstraintsRequest(DEFAULT_CATALOG_NAME, dbName+ "_dupe" , "tbl3")); assertEquals(nns.size(), 1); } catch (TException te) { assertNull(te); @@ -3104,13 +3105,13 @@ public class TestReplicationScenarios { List<SQLPrimaryKey> pks = metaStoreClientMirror.getPrimaryKeys(new PrimaryKeysRequest(dbName+ "_dupe" , "tbl4")); assertEquals(pks.size(), 2); pkName = pks.get(0).getPk_name(); - List<SQLUniqueConstraint> uks = metaStoreClientMirror.getUniqueConstraints(new UniqueConstraintsRequest(dbName+ "_dupe" , "tbl6")); + List<SQLUniqueConstraint> uks = metaStoreClientMirror.getUniqueConstraints(new UniqueConstraintsRequest(DEFAULT_CATALOG_NAME, dbName+ "_dupe" , "tbl6")); assertEquals(uks.size(), 1); ukName = uks.get(0).getUk_name(); List<SQLForeignKey> fks = metaStoreClientMirror.getForeignKeys(new ForeignKeysRequest(null, null, dbName+ "_dupe" , "tbl5")); assertEquals(fks.size(), 2); fkName = fks.get(0).getFk_name(); - List<SQLNotNullConstraint> nns = metaStoreClientMirror.getNotNullConstraints(new NotNullConstraintsRequest(dbName+ "_dupe" , "tbl6")); + List<SQLNotNullConstraint> nns = metaStoreClientMirror.getNotNullConstraints(new NotNullConstraintsRequest(DEFAULT_CATALOG_NAME, dbName+ "_dupe" , "tbl6")); assertEquals(nns.size(), 1); nnName = nns.get(0).getNn_name(); @@ -3133,11 +3134,11 @@ public class TestReplicationScenarios { try { List<SQLPrimaryKey> pks = metaStoreClientMirror.getPrimaryKeys(new PrimaryKeysRequest(dbName+ "_dupe" , "tbl4")); assertTrue(pks.isEmpty()); - List<SQLUniqueConstraint> uks = metaStoreClientMirror.getUniqueConstraints(new UniqueConstraintsRequest(dbName+ "_dupe" , "tbl4")); + List<SQLUniqueConstraint> uks = metaStoreClientMirror.getUniqueConstraints(new UniqueConstraintsRequest(DEFAULT_CATALOG_NAME, dbName+ "_dupe" , "tbl4")); assertTrue(uks.isEmpty()); List<SQLForeignKey> fks = metaStoreClientMirror.getForeignKeys(new ForeignKeysRequest(null, null, dbName+ "_dupe" , "tbl5")); assertTrue(fks.isEmpty()); - List<SQLNotNullConstraint> nns = metaStoreClientMirror.getNotNullConstraints(new NotNullConstraintsRequest(dbName+ "_dupe" , "tbl6")); + List<SQLNotNullConstraint> nns = metaStoreClientMirror.getNotNullConstraints(new NotNullConstraintsRequest(DEFAULT_CATALOG_NAME, dbName+ "_dupe" , "tbl6")); assertTrue(nns.isEmpty()); } catch (TException te) { assertNull(te); http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/metastore/scripts/upgrade/derby/053-HIVE-18755.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/053-HIVE-18755.derby.sql b/metastore/scripts/upgrade/derby/053-HIVE-18755.derby.sql new file mode 100644 index 0000000..776ef14 --- /dev/null +++ b/metastore/scripts/upgrade/derby/053-HIVE-18755.derby.sql @@ -0,0 +1,54 @@ + +CREATE TABLE "APP"."CTLGS" ( + "CTLG_ID" BIGINT NOT NULL, + "NAME" VARCHAR(256) UNIQUE, + "DESC" VARCHAR(4000), + "LOCATION_URI" VARCHAR(4000) NOT NULL); + +ALTER TABLE "APP"."CTLGS" ADD CONSTRAINT "CTLGS_PK" PRIMARY KEY ("CTLG_ID"); + +-- Insert a default value. The location is TBD. Hive will fix this when it starts +INSERT INTO "APP"."CTLGS" VALUES (1, 'hive', 'Default catalog for Hive', 'TBD'); + +-- Drop the unique index on DBS +DROP INDEX "APP"."UNIQUE_DATABASE"; + +-- Add the new column to the DBS table, can't put in the not null constraint yet +ALTER TABLE "APP"."DBS" ADD COLUMN "CTLG_NAME" VARCHAR(256); + +-- Update all records in the DBS table to point to the Hive catalog +UPDATE "APP"."DBS" + SET "CTLG_NAME" = 'hive'; + +-- Add the not null constraint +ALTER TABLE "APP"."DBS" ALTER COLUMN "CTLG_NAME" NOT NULL; + +-- Put back the unique index +CREATE UNIQUE INDEX "APP"."UNIQUE_DATABASE" ON "APP"."DBS" ("NAME", "CTLG_NAME"); + +-- Add the foreign key +ALTER TABLE "APP"."DBS" ADD CONSTRAINT "DBS_FK1" FOREIGN KEY ("CTLG_NAME") REFERENCES "APP"."CTLGS" ("NAME") ON DELETE NO ACTION ON UPDATE NO ACTION; + +-- Add columns to table stats and part stats +ALTER TABLE "APP"."TAB_COL_STATS" ADD COLUMN "CAT_NAME" VARCHAR(256); +ALTER TABLE "APP"."PART_COL_STATS" ADD COLUMN "CAT_NAME" VARCHAR(256); + +-- Set the existing column names to Hive +UPDATE "APP"."TAB_COL_STATS" + SET "CAT_NAME" = 'hive'; +UPDATE "APP"."PART_COL_STATS" + SET "CAT_NAME" = 'hive'; + +-- Add the not null constraint +ALTER TABLE "APP"."TAB_COL_STATS" ALTER COLUMN "CAT_NAME" NOT NULL; +ALTER TABLE "APP"."PART_COL_STATS" ALTER COLUMN "CAT_NAME" NOT NULL; + +-- Rebuild the index for Part col stats. No such index for table stats, which seems weird +DROP INDEX "APP"."PCS_STATS_IDX"; +CREATE INDEX "APP"."PCS_STATS_IDX" ON "APP"."PART_COL_STATS" ("CAT_NAME", "DB_NAME","TABLE_NAME","COLUMN_NAME","PARTITION_NAME"); + +-- Add column to partition events +ALTER TABLE "APP"."PARTITION_EVENTS" ADD COLUMN "CAT_NAME" VARCHAR(256); + +-- Add column to notification log +ALTER TABLE "APP"."NOTIFICATION_LOG" ADD COLUMN "CAT_NAME" VARCHAR(256); http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql b/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql index 1e4dd99..1a3c00a 100644 --- a/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql +++ b/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql @@ -10,5 +10,6 @@ RUN '049-HIVE-18489.derby.sql'; RUN '050-HIVE-18192.derby.sql'; RUN '051-HIVE-18675.derby.sql'; RUN '052-HIVE-18965.derby.sql'; +RUN '053-HIVE-18755.derby.sql'; UPDATE "APP".VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 01cadea..429c296 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.exec; import static org.apache.commons.lang.StringUtils.join; +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE; import java.io.BufferedWriter; @@ -5028,8 +5029,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable { // We set the signature for the view if it is a materialized view if (tbl.isMaterializedView()) { CreationMetadata cm = - new CreationMetadata(tbl.getDbName(), tbl.getTableName(), - ImmutableSet.copyOf(crtView.getTablesUsed())); + new CreationMetadata(MetaStoreUtils.getDefaultCatalog(conf), tbl.getDbName(), + tbl.getTableName(), ImmutableSet.copyOf(crtView.getTablesUsed())); cm.setValidTxnList(conf.get(ValidTxnList.VALID_TXNS_KEY)); tbl.getTTable().setCreationMetadata(cm); } http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewTask.java index de120af..50fc4e0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewTask.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.exec; import com.google.common.collect.ImmutableSet; import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.metastore.api.CreationMetadata; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -31,6 +32,8 @@ import org.apache.hadoop.hive.ql.plan.api.StageType; import java.io.Serializable; +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; + /** * This task does some work related to materialized views. In particular, it adds * or removes the materialized view from the registry if needed, or registers new @@ -63,7 +66,8 @@ public class MaterializedViewTask extends Task<MaterializedViewDesc> implements Hive db = Hive.get(conf); Table mvTable = db.getTable(getWork().getViewName()); CreationMetadata cm = - new CreationMetadata(mvTable.getDbName(), mvTable.getTableName(), + new CreationMetadata(MetaStoreUtils.getDefaultCatalog(conf), mvTable.getDbName(), + mvTable.getTableName(), ImmutableSet.copyOf(mvTable.getCreationMetadata().getTablesUsed())); cm.setValidTxnList(conf.get(ValidTxnList.VALID_TXNS_KEY)); db.updateCreationMetadata(mvTable.getDbName(), mvTable.getTableName(), cm); http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index d76c8b6..eed37a1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -25,7 +25,9 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; import static org.apache.hadoop.hive.conf.Constants.MATERIALIZED_VIEW_REWRITING_TIME_WINDOW; + import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT; import static org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME; @@ -4491,7 +4493,7 @@ private void constructOneLBLocationMap(FileStatus fSta, public List<SQLUniqueConstraint> getUniqueConstraintList(String dbName, String tblName) throws HiveException, NoSuchObjectException { try { - return getMSC().getUniqueConstraints(new UniqueConstraintsRequest(dbName, tblName)); + return getMSC().getUniqueConstraints(new UniqueConstraintsRequest(getDefaultCatalog(conf), dbName, tblName)); } catch (NoSuchObjectException e) { throw e; } catch (Exception e) { @@ -4501,7 +4503,7 @@ private void constructOneLBLocationMap(FileStatus fSta, public List<SQLNotNullConstraint> getNotNullConstraintList(String dbName, String tblName) throws HiveException, NoSuchObjectException { try { - return getMSC().getNotNullConstraints(new NotNullConstraintsRequest(dbName, tblName)); + return getMSC().getNotNullConstraints(new NotNullConstraintsRequest(getDefaultCatalog(conf), dbName, tblName)); } catch (NoSuchObjectException e) { throw e; } catch (Exception e) { @@ -4511,7 +4513,7 @@ private void constructOneLBLocationMap(FileStatus fSta, public List<SQLDefaultConstraint> getDefaultConstraintList(String dbName, String tblName) throws HiveException, NoSuchObjectException { try { - return getMSC().getDefaultConstraints(new DefaultConstraintsRequest(dbName, tblName)); + return getMSC().getDefaultConstraints(new DefaultConstraintsRequest(getDefaultCatalog(conf), dbName, tblName)); } catch (NoSuchObjectException e) { throw e; } catch (Exception e) { @@ -4521,7 +4523,8 @@ private void constructOneLBLocationMap(FileStatus fSta, public List<SQLCheckConstraint> getCheckConstraintList(String dbName, String tblName) throws HiveException, NoSuchObjectException { try { - return getMSC().getCheckConstraints(new CheckConstraintsRequest(dbName, tblName)); + return getMSC().getCheckConstraints(new CheckConstraintsRequest(getDefaultCatalog(conf), + dbName, tblName)); } catch (NoSuchObjectException e) { throw e; } catch (Exception e) { @@ -4635,7 +4638,7 @@ private void constructOneLBLocationMap(FileStatus fSta, throws HiveException { try { List<SQLUniqueConstraint> uniqueConstraints = getMSC().getUniqueConstraints( - new UniqueConstraintsRequest(dbName, tblName)); + new UniqueConstraintsRequest(getDefaultCatalog(conf), dbName, tblName)); if (onlyReliable && uniqueConstraints != null && !uniqueConstraints.isEmpty()) { uniqueConstraints = uniqueConstraints.stream() .filter(uk -> uk.isRely_cstr()) @@ -4683,7 +4686,7 @@ private void constructOneLBLocationMap(FileStatus fSta, throws HiveException { try { List<SQLNotNullConstraint> notNullConstraints = getMSC().getNotNullConstraints( - new NotNullConstraintsRequest(dbName, tblName)); + new NotNullConstraintsRequest(getDefaultCatalog(conf), dbName, tblName)); if (notNullConstraints != null && !notNullConstraints.isEmpty()) { notNullConstraints = notNullConstraints.stream() .filter(nnc -> nnc.isEnable_cstr()) @@ -4707,7 +4710,7 @@ private void constructOneLBLocationMap(FileStatus fSta, throws HiveException { try { List<SQLCheckConstraint> checkConstraints = getMSC().getCheckConstraints( - new CheckConstraintsRequest(dbName, tblName)); + new CheckConstraintsRequest(getDefaultCatalog(conf), dbName, tblName)); if (checkConstraints != null && !checkConstraints.isEmpty()) { checkConstraints = checkConstraints.stream() .filter(nnc -> nnc.isEnable_cstr()) @@ -4730,7 +4733,7 @@ private void constructOneLBLocationMap(FileStatus fSta, throws HiveException { try { List<SQLDefaultConstraint> defaultConstraints = getMSC().getDefaultConstraints( - new DefaultConstraintsRequest(dbName, tblName)); + new DefaultConstraintsRequest(getDefaultCatalog(conf), dbName, tblName)); if (defaultConstraints != null && !defaultConstraints.isEmpty()) { defaultConstraints = defaultConstraints.stream() .filter(nnc -> nnc.isEnable_cstr()) @@ -4746,7 +4749,7 @@ private void constructOneLBLocationMap(FileStatus fSta, throws HiveException { try { List<SQLNotNullConstraint> notNullConstraints = getMSC().getNotNullConstraints( - new NotNullConstraintsRequest(dbName, tblName)); + new NotNullConstraintsRequest(getDefaultCatalog(conf), dbName, tblName)); if (onlyReliable && notNullConstraints != null && !notNullConstraints.isEmpty()) { notNullConstraints = notNullConstraints.stream() .filter(nnc -> nnc.isRely_cstr()) @@ -4762,7 +4765,7 @@ private void constructOneLBLocationMap(FileStatus fSta, throws HiveException { try { List<SQLDefaultConstraint> defaultConstraints = getMSC().getDefaultConstraints( - new DefaultConstraintsRequest(dbName, tblName)); + new DefaultConstraintsRequest(getDefaultCatalog(conf), dbName, tblName)); if (defaultConstraints != null && !defaultConstraints.isEmpty()) { defaultConstraints = defaultConstraints.stream() .collect(Collectors.toList()); @@ -4777,7 +4780,7 @@ private void constructOneLBLocationMap(FileStatus fSta, throws HiveException { try { List<SQLCheckConstraint> checkConstraints = getMSC().getCheckConstraints( - new CheckConstraintsRequest(dbName, tblName)); + new CheckConstraintsRequest(getDefaultCatalog(conf), dbName, tblName)); if (checkConstraints != null && !checkConstraints.isEmpty()) { checkConstraints = checkConstraints.stream() .collect(Collectors.toList());
