Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign e5bb466ef -> c56f48cc6
http://git-wip-us.apache.org/repos/asf/sentry/blob/c56f48cc/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java deleted file mode 100644 index c6c9448..0000000 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java +++ /dev/null @@ -1,465 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.service.thrift; - -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import java.util.ArrayList; -import java.util.List; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.NotificationEvent; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hive.hcatalog.messaging.HCatEventMessage; -import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory; -import org.apache.sentry.hdfs.Updateable; -import org.apache.sentry.provider.db.service.persistent.SentryStore; -import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable; -import org.junit.After; -import org.junit.BeforeClass; -import org.junit.Test; -import org.mockito.Mockito; - -// TODO 1. More tests should be added here. -// TODO 2. Tests using actual sentry store where. -@SuppressWarnings("unused") -public class TestNotificationProcessor { - - private static final SentryStore sentryStore = Mockito.mock(SentryStore.class); - private final static String hiveInstance = "server2"; - private final static Configuration conf = new Configuration(); - private final SentryJSONMessageFactory messageFactory = new SentryJSONMessageFactory(); - private NotificationProcessor notificationProcessor; - - @BeforeClass - public static void setup() { - conf.set("sentry.hive.sync.create", "true"); - conf.set("sentry.hive.sync.drop", "true"); - } - - @After - public void resetConf() { - conf.set("sentry.hive.sync.create", "true"); - conf.set("sentry.hive.sync.drop", "true"); - reset(sentryStore); - } - - @Test - /* - Makes sure that appropriate sentry store methods are invoked when create database event is - processed. - - Also, checks the hive sync configuration. - */ - public void testCreateDatabase() throws Exception { - long seqNum = 1; - String dbName = "db1"; - String uriPrefix = "hdfs:///"; - String location = "user/hive/warehouse"; - NotificationEvent notificationEvent; - TSentryAuthorizable authorizable; - notificationProcessor = new NotificationProcessor(sentryStore, - hiveInstance, conf); - - // Create notification event - notificationEvent = new NotificationEvent(seqNum, 0, - HCatEventMessage.EventType.CREATE_DATABASE.toString(), - messageFactory.buildCreateDatabaseMessage(new Database(dbName, - null, uriPrefix + location, null)).toString()); - - notificationProcessor.processNotificationEvent(notificationEvent); - - authorizable = new TSentryAuthorizable(hiveInstance); - authorizable.setServer(hiveInstance); - authorizable.setDb("db1"); - //noinspection unchecked - verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); - - verify(sentryStore, times(1)).dropPrivilege(authorizable, - NotificationProcessor.getPermUpdatableOnDrop(authorizable)); - reset(sentryStore); - - //Change the configuration and make sure that exiting privileges are not dropped - notificationProcessor.setSyncStoreOnCreate(false); - dbName = "db2"; - notificationEvent = new NotificationEvent(1, 0, - HCatEventMessage.EventType.CREATE_DATABASE.toString(), - messageFactory.buildCreateDatabaseMessage(new Database(dbName, - null, "hdfs:///db2", null)).toString()); - - notificationProcessor.processNotificationEvent(notificationEvent); - - authorizable = new TSentryAuthorizable(hiveInstance); - authorizable.setServer(hiveInstance); - authorizable.setDb(dbName); - - //noinspection unchecked - verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); - //making sure that privileges are not dropped - verify(sentryStore, times(0)).dropPrivilege(authorizable, - NotificationProcessor.getPermUpdatableOnDrop(authorizable)); - - } - - @Test - /* - Makes sure that appropriate sentry store methods are invoked when drop database event is - processed. - - Also, checks the hive sync configuration. - */ - public void testDropDatabase() throws Exception { - String dbName = "db1"; - - notificationProcessor = new NotificationProcessor(sentryStore, - hiveInstance, conf); - - // Create notification event - NotificationEvent notificationEvent = new NotificationEvent(1, 0, - HCatEventMessage.EventType.DROP_DATABASE.toString(), - messageFactory.buildDropDatabaseMessage(new Database(dbName, null, - "hdfs:///db1", null)).toString()); - - notificationProcessor.processNotificationEvent(notificationEvent); - - TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); - authorizable.setServer(hiveInstance); - authorizable.setDb("db1"); - - //noinspection unchecked - verify(sentryStore, times(1)).deleteAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); - verify(sentryStore, times(1)).dropPrivilege(authorizable, - NotificationProcessor.getPermUpdatableOnDrop(authorizable)); - reset(sentryStore); - - // Change the configuration and make sure that exiting privileges are not dropped - notificationProcessor.setSyncStoreOnDrop(false); - dbName = "db2"; - // Create notification event - notificationEvent = new NotificationEvent(1, 0, - HCatEventMessage.EventType.DROP_DATABASE.toString(), - messageFactory.buildDropDatabaseMessage(new Database(dbName, null, - "hdfs:///db2", null)).toString()); - - notificationProcessor.processNotificationEvent(notificationEvent); - - authorizable = new TSentryAuthorizable(hiveInstance); - authorizable.setServer(hiveInstance); - authorizable.setDb(dbName); - - //noinspection unchecked - verify(sentryStore, times(1)).deleteAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); - verify(sentryStore, times(0)).dropPrivilege(authorizable, - NotificationProcessor.getPermUpdatableOnDrop(authorizable)); - } - - @Test - /* - Makes sure that appropriate sentry store methods are invoked when create table event is - processed. - - Also, checks the hive sync configuration. - */ - public void testCreateTable() throws Exception { - String dbName = "db1"; - String tableName = "table1"; - - notificationProcessor = new NotificationProcessor(sentryStore, - hiveInstance, conf); - - // Create notification event - StorageDescriptor sd = new StorageDescriptor(); - sd.setLocation("hdfs:///db1.db/table1"); - NotificationEvent notificationEvent = - new NotificationEvent(1, 0, HCatEventMessage.EventType.CREATE_TABLE.toString(), - messageFactory.buildCreateTableMessage(new Table(tableName, - dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString()); - - notificationProcessor.processNotificationEvent(notificationEvent); - - TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); - authorizable.setServer(hiveInstance); - authorizable.setDb("db1"); - authorizable.setTable(tableName); - - //noinspection unchecked - verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); - - verify(sentryStore, times(1)).dropPrivilege(authorizable, - NotificationProcessor.getPermUpdatableOnDrop(authorizable)); - reset(sentryStore); - - // Change the configuration and make sure that existing privileges are not dropped - notificationProcessor.setSyncStoreOnCreate(false); - - // Create notification event - dbName = "db2"; - tableName = "table2"; - sd = new StorageDescriptor(); - sd.setLocation("hdfs:///db1.db/table2"); - notificationEvent = - new NotificationEvent(1, 0, HCatEventMessage.EventType.CREATE_TABLE.toString(), - messageFactory.buildCreateTableMessage(new Table(tableName, - dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString()); - - notificationProcessor.processNotificationEvent(notificationEvent); - - authorizable = new TSentryAuthorizable(hiveInstance); - authorizable.setServer(hiveInstance); - authorizable.setDb(dbName); - authorizable.setTable(tableName); - - //noinspection unchecked - verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); - // Making sure that privileges are not dropped - verify(sentryStore, times(0)).dropPrivilege(authorizable, - NotificationProcessor.getPermUpdatableOnDrop(authorizable)); - } - - @Test - /* - Makes sure that appropriate sentry store methods are invoked when drop table event is - processed. - - Also, checks the hive sync configuration. - */ - public void testDropTable() throws Exception { - String dbName = "db1"; - String tableName = "table1"; - - Configuration authConf = new Configuration(); - notificationProcessor = new NotificationProcessor(sentryStore, - hiveInstance, authConf); - - // Create notification event - StorageDescriptor sd = new StorageDescriptor(); - sd.setLocation("hdfs:///db1.db/table1"); - NotificationEvent notificationEvent = new NotificationEvent(1, 0, - HCatEventMessage.EventType.DROP_TABLE.toString(), - messageFactory.buildDropTableMessage(new Table(tableName, - dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString()); - - notificationProcessor.processNotificationEvent(notificationEvent); - - TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); - authorizable.setServer(hiveInstance); - authorizable.setDb("db1"); - authorizable.setTable(tableName); - - verify(sentryStore, times(1)).deleteAllAuthzPathsMapping(Mockito.anyString(), - Mockito.any(Updateable.Update.class)); - - verify(sentryStore, times(1)).dropPrivilege(authorizable, - NotificationProcessor.getPermUpdatableOnDrop(authorizable)); - } - - @Test - /* - Makes sure that appropriate sentry store methods are invoked when alter tables event is - processed. - */ - public void testAlterTable() throws Exception { - String dbName = "db1"; - String tableName = "table1"; - - String newDbName = "db1"; - String newTableName = "table2"; - - Configuration authConf = new Configuration(); - notificationProcessor = new NotificationProcessor(sentryStore, - hiveInstance, authConf); - - // Create notification event - StorageDescriptor sd = new StorageDescriptor(); - sd.setLocation("hdfs:///db1.db/table1"); - NotificationEvent notificationEvent = new NotificationEvent(1, 0, - HCatEventMessage.EventType.ALTER_TABLE.toString(), - messageFactory.buildAlterTableMessage( - new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null), - new Table(newTableName, newDbName, null, 0, 0, 0, sd, null, null, null, null, null)) - .toString()); - notificationEvent.setDbName(newDbName); - notificationEvent.setTableName(newTableName); - - notificationProcessor.processNotificationEvent(notificationEvent); - - TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); - authorizable.setServer(hiveInstance); - authorizable.setDb(dbName); - authorizable.setTable(tableName); - - TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance); - authorizable.setServer(hiveInstance); - newAuthorizable.setDb(newDbName); - newAuthorizable.setTable(newTableName); - - verify(sentryStore, times(1)).renameAuthzObj(Mockito.anyString(), Mockito.anyString(), - Mockito.any(Updateable.Update.class)); - - verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable, - NotificationProcessor.getPermUpdatableOnRename(authorizable, newAuthorizable)); - } - - @Test - /* - Makes sure that appropriate sentry store methods are invoked when alter tables event is - processed. - */ - public void testRenameTableWithLocationUpdate() throws Exception { - String dbName = "db1"; - String tableName = "table1"; - - String newDbName = "db1"; - String newTableName = "table2"; - - Configuration authConf = new Configuration(); - notificationProcessor = new NotificationProcessor(sentryStore, - hiveInstance, authConf); - - // Create notification event - StorageDescriptor sd = new StorageDescriptor(); - sd.setLocation("hdfs:///db1.db/table1"); - StorageDescriptor new_sd = new StorageDescriptor(); - new_sd.setLocation("hdfs:///db1.db/table2"); - NotificationEvent notificationEvent = new NotificationEvent(1, 0, - HCatEventMessage.EventType.ALTER_TABLE.toString(), - messageFactory.buildAlterTableMessage( - new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null), - new Table(newTableName, newDbName, null, 0, 0, 0, new_sd, null, null, null, null, null)) - .toString()); - notificationEvent.setDbName(newDbName); - notificationEvent.setTableName(newTableName); - - notificationProcessor.processNotificationEvent(notificationEvent); - - TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); - authorizable.setServer(hiveInstance); - authorizable.setDb(dbName); - authorizable.setTable(tableName); - - TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance); - authorizable.setServer(hiveInstance); - newAuthorizable.setDb(newDbName); - newAuthorizable.setTable(newTableName); - - verify(sentryStore, times(1)).renameAuthzPathsMapping(Mockito.anyString(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), Mockito.any(Updateable.Update.class)); - - verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable, - NotificationProcessor.getPermUpdatableOnRename(authorizable, newAuthorizable)); - } - - @Test - /* - Test to made sure that sentry store is not invoked when invalid alter table event is - processed. - */ - public void testAlterTableWithInvalidEvent() throws Exception { - String dbName = "db1"; - String tableName1 = "table1"; - String tableName2 = "table2"; - long inputEventId = 1; - NotificationEvent notificationEvent; - List<FieldSchema> partCols; - StorageDescriptor sd; - Mockito.doNothing().when(sentryStore).persistLastProcessedNotificationID(Mockito.anyLong()); - //noinspection unchecked - Mockito.doNothing().when(sentryStore).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); - - Configuration authConf = new Configuration(); - notificationProcessor = new NotificationProcessor(sentryStore, - hiveInstance, authConf); - - // Create a table - sd = new StorageDescriptor(); - sd.setLocation("hdfs://db1.db/table1"); - partCols = new ArrayList<>(); - partCols.add(new FieldSchema("ds", "string", "")); - Table table = new Table(tableName1, dbName, null, 0, 0, 0, sd, partCols, - null, null, null, null); - notificationEvent = new NotificationEvent(inputEventId, 0, - HCatEventMessage.EventType.CREATE_TABLE.toString(), - messageFactory.buildCreateTableMessage(table).toString()); - notificationEvent.setDbName(dbName); - notificationEvent.setTableName(tableName1); - inputEventId += 1; - // Process the notification - notificationProcessor.processNotificationEvent(notificationEvent); - // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification - // and persistLastProcessedNotificationID was not invoked. - //noinspection unchecked - verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); - reset(sentryStore); - - // Create alter table notification with out actually changing anything. - // This notification should not be processed by sentry server - // Notification should be persisted explicitly - notificationEvent = new NotificationEvent(1, 0, - HCatEventMessage.EventType.ALTER_TABLE.toString(), - messageFactory.buildAlterTableMessage( - new Table(tableName1, dbName, null, 0, 0, 0, sd, null, null, null, null, null), - new Table(tableName1, dbName, null, 0, 0, 0, sd, null, - null, null, null, null)).toString()); - notificationEvent.setDbName(dbName); - notificationEvent.setTableName(tableName1); - inputEventId += 1; - // Process the notification - notificationProcessor.processNotificationEvent(notificationEvent); - // Make sure that renameAuthzObj and deleteAuthzPathsMapping were not invoked - // to handle CREATE_TABLE notification - // and persistLastProcessedNotificationID is explicitly invoked - verify(sentryStore, times(0)).renameAuthzObj(Mockito.anyString(), Mockito.anyString(), - Mockito.any(Updateable.Update.class)); - //noinspection unchecked - verify(sentryStore, times(0)).deleteAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); - reset(sentryStore); - - // Create a table - sd = new StorageDescriptor(); - sd.setLocation("hdfs://db1.db/table2"); - partCols = new ArrayList<>(); - partCols.add(new FieldSchema("ds", "string", "")); - Table table1 = new Table(tableName2, dbName, null, 0, 0, 0, sd, - partCols, null, null, null, null); - notificationEvent = new NotificationEvent(inputEventId, 0, - HCatEventMessage.EventType.CREATE_TABLE.toString(), - messageFactory.buildCreateTableMessage(table1).toString()); - notificationEvent.setDbName(dbName); - notificationEvent.setTableName(tableName2); - // Process the notification - notificationProcessor.processNotificationEvent(notificationEvent); - // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification - // and persistLastProcessedNotificationID was not invoked. - //noinspection unchecked - verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), - Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/c56f48cc/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java deleted file mode 100644 index 3cc6541..0000000 --- a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java +++ /dev/null @@ -1,470 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.service.thrift; - -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Random; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.IMetaStoreClient.NotificationFilter; -import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.NotificationEvent; -import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hive.hcatalog.messaging.HCatEventMessage; -import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory; -import org.apache.sentry.provider.db.service.persistent.PathsImage; -import org.apache.thrift.TException; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -import javax.security.auth.login.LoginException; - -/** - * Test mocks HiveMetaStoreClient class and tests SentryHmsClient. - */ -public class TestSentryHmsClient { - - private static final Configuration conf = new Configuration(); - private static final SentryJSONMessageFactory messageFactory = new SentryJSONMessageFactory(); - private static SentryHmsClient client; - private static MockHMSClientFactory hiveConnectionFactory; - - /** - * Create mock database with the given name - * - * @param name Database name - * @return Mock database object - */ - private static Database makeDb(String name) { - Database db = Mockito.mock(Database.class); - Mockito.when(db.getName()).thenReturn(name); - Mockito.when(db.getLocationUri()).thenReturn("hdfs:///" + name); - return db; - } - - /** - * Create mock table - * - * @param dbName db for this table - * @param tableName name of the table - * @return mock table object - */ - private static Table makeTable(String dbName, String tableName) { - Table table = Mockito.mock(Table.class); - Mockito.when(table.getDbName()).thenReturn(dbName); - Mockito.when(table.getTableName()).thenReturn(tableName); - StorageDescriptor sd = Mockito.mock(StorageDescriptor.class); - Mockito.when(sd.getLocation()).thenReturn( - String.format("hdfs:///%s/%s", dbName, tableName)); - Mockito.when(table.getSd()).thenReturn(sd); - return table; - } - - /** - * Create mock partition - * - * @param dbName database for this partition - * @param tableName table for this partition - * @param partName partition name - * @return mock partition object - */ - private static Partition makePartition(String dbName, String tableName, String partName) { - Partition partition = Mockito.mock(Partition.class); - StorageDescriptor sd = Mockito.mock(StorageDescriptor.class); - Mockito.when(sd.getLocation()).thenReturn( - String.format("hdfs:///%s/%s/%s", dbName, tableName, partName)); - Mockito.when(partition.getSd()).thenReturn(sd); - return partition; - } - - /** - * Creates create database notification - * - * @return NotificationEvent - */ - private static NotificationEvent getCreateDatabaseNotification(long id) { - Random rand = new Random(); - int n = rand.nextInt(100) + 1; - String dbName = "db" + n; - return new NotificationEvent(id, 0, HCatEventMessage.EventType.CREATE_DATABASE.toString(), - messageFactory - .buildCreateDatabaseMessage(new Database(dbName, null, "hdfs:///" + dbName, null)) - .toString()); - } - - /** - * Creates drop database notification - * - * @return NotificationEvent - */ - private static NotificationEvent getDropDatabaseNotification(long id) { - Random rand = new Random(); - int n = rand.nextInt(100) + 1; - String dbName = "db" + n; - return new NotificationEvent(id, 0, HCatEventMessage.EventType.DROP_DATABASE.toString(), - messageFactory - .buildDropDatabaseMessage(new Database(dbName, null, "hdfs:///" + dbName, null)) - .toString()); - } - - @BeforeClass - static public void initialize() throws IOException, LoginException { - hiveConnectionFactory = new MockHMSClientFactory(); - client = new SentryHmsClient(conf, (HiveConnectionFactory)hiveConnectionFactory); - } - - /** - * Creating snapshot when SentryHmsClient is not connected to HMS - */ - @Test - public void testSnapshotCreationWithOutClientConnected() throws Exception { - // Make sure that client is not connected - Assert.assertFalse(client.isConnected()); - PathsImage snapshotInfo = client.getFullSnapshot(); - Assert.assertTrue(snapshotInfo.getPathImage().isEmpty()); - } - - /** - * Creating snapshot when HMS doesn't have any data - */ - @Test - public void testSnapshotCreationWithNoHmsData() throws Exception { - MockClient mockClient = new MockClient(new HiveSnapshot(), 1); - client.setClient(mockClient.client); - // Make sure that client is connected - Assert.assertTrue(client.isConnected()); - PathsImage snapshotInfo = client.getFullSnapshot(); - Assert.assertTrue(snapshotInfo.getPathImage().isEmpty()); - } - - /** - * Creating a snapshot when there is data but there are updates to HMS data mean while - */ - @Test - public void testSnapshotCreationWhenDataIsActivelyUpdated() throws Exception { - HiveTable tab21 = new HiveTable("tab21"); - HiveTable tab31 = new HiveTable("tab31").add("part311").add("part312"); - HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31)); - HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21)); - HiveDb db1 = new HiveDb("db1"); - HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3); - final MockClient mockClient = new MockClient(snap, 1); - - client.setClient(mockClient.client); - hiveConnectionFactory.setClient(mockClient); - // Make sure that client is connected - Assert.assertTrue(client.isConnected()); - PathsImage snapshotInfo = client.getFullSnapshot(); - // Make sure that snapshot is not empty - Assert.assertTrue(!snapshotInfo.getPathImage().isEmpty()); - - Mockito.when(mockClient.client.getCurrentNotificationEventId()). - thenAnswer(new Answer<CurrentNotificationEventId>() { - @Override - public CurrentNotificationEventId answer(InvocationOnMock invocation) - throws Throwable { - return new CurrentNotificationEventId(mockClient.incrementNotificationEventId()); - } - - }); - - snapshotInfo = client.getFullSnapshot(); - Assert.assertTrue(snapshotInfo.getPathImage().isEmpty()); - } - - /** - * Creating a snapshot when there is data in HMS. - */ - @Test - public void testSnapshotCreationSuccess() throws Exception { - HiveTable tab21 = new HiveTable("tab21"); - HiveTable tab31 = new HiveTable("tab31"); - HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31)); - HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21)); - HiveDb db1 = new HiveDb("db1"); - HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3); - MockClient mockClient = new MockClient(snap, 1); - Mockito.when(mockClient.client.getCurrentNotificationEventId()). - thenReturn(new CurrentNotificationEventId(mockClient.eventId)); - client.setClient(mockClient.client); - hiveConnectionFactory.setClient(mockClient); - // Make sure that client is connected - Assert.assertTrue(client.isConnected()); - - PathsImage snapshotInfo = client.getFullSnapshot(); - Assert.assertEquals(5, snapshotInfo.getPathImage().size()); - Assert.assertEquals(Sets.newHashSet("db1"), snapshotInfo.getPathImage().get("db1")); - Assert.assertEquals(Sets.newHashSet("db2"), snapshotInfo.getPathImage().get("db2")); - Assert.assertEquals(Sets.newHashSet("db3"), snapshotInfo.getPathImage().get("db3")); - Assert.assertEquals(Sets.newHashSet("db2/tab21"), - snapshotInfo.getPathImage().get("db2.tab21")); - Assert.assertEquals(Sets.newHashSet("db3/tab31"), snapshotInfo.getPathImage().get("db3.tab31")); - Assert.assertEquals(snapshotInfo.getId(), mockClient.eventId); - - } - - /** - * Test scenario when there is no HMS connection - * Getting new notifications - */ - @Test - public void testGetNewNotificationsWithOutClientConnected() throws Exception { - HiveTable tab21 = new HiveTable("tab21"); - HiveTable tab31 = new HiveTable("tab31"); - HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31)); - HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21)); - HiveDb db1 = new HiveDb("db1"); - client.setClient(null); - HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3); - MockClient mockClient = new MockClient(snap, 100); - Mockito.when(mockClient.client.getCurrentNotificationEventId()). - thenReturn(new CurrentNotificationEventId(mockClient.eventId)); - // Make sure that client is not connected - Assert.assertTrue(!client.isConnected()); - Collection<NotificationEvent> events = client.getNotifications(100); - Assert.assertTrue(events.isEmpty()); - - } - - /** - * Test scenario where there are no notifications - * Getting new notifications - */ - @Test - public void testGetNewNotificationsWithNoHmsUpdates() throws Exception { - HiveTable tab21 = new HiveTable("tab21"); - HiveTable tab31 = new HiveTable("tab31"); - HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31)); - HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21)); - HiveDb db1 = new HiveDb("db1"); - HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3); - MockClient mockClient = new MockClient(snap, 100); - Mockito.when(mockClient.client.getCurrentNotificationEventId()). - thenReturn(new CurrentNotificationEventId(mockClient.eventId)); - client.setClient(mockClient.client); - hiveConnectionFactory.setClient(mockClient); - // Make sure that client is connected - Assert.assertTrue(client.isConnected()); - Collection<NotificationEvent> events = client.getNotifications(100); - Assert.assertTrue(events.isEmpty()); - } - - /** - * Test scenario where there are notifications - * Getting new notifications - */ - @Test - public void testGetNewNotificationsSuccess() throws Exception { - final MockClient mockClient = new MockClient(new HiveSnapshot(), 100); - client.setClient(mockClient.client); - hiveConnectionFactory.setClient(mockClient); - // Make sure that client is connected - Assert.assertTrue(client.isConnected()); - - Mockito.when(mockClient.client.getCurrentNotificationEventId()). - thenAnswer(new Answer<CurrentNotificationEventId>() { - @Override - public CurrentNotificationEventId answer(InvocationOnMock invocation) - throws Throwable { - return new CurrentNotificationEventId(mockClient.incrementNotificationEventId()); - } - }); - Mockito.when(mockClient.client.getNextNotification(Mockito.anyLong(), Mockito.anyInt(), - Mockito.any(NotificationFilter.class))). - thenAnswer(new Answer<NotificationEventResponse>() { - @Override - public NotificationEventResponse answer(InvocationOnMock invocation) - throws Throwable { - long id = 1; - List<NotificationEvent> events = new ArrayList<>(); - events.add(getCreateDatabaseNotification(id++)); - events.add(getDropDatabaseNotification(id++)); - return new NotificationEventResponse(events); - } - }); - - Collection<NotificationEvent> events = client.getNotifications(100); - long id = 1; - for (NotificationEvent event : events) { - Assert.assertEquals(event.getEventId(), id++); - } - Assert.assertTrue(events.size() == 2); - } - - /** - * Representation of a Hive table. A table has a name and a list of partitions. - */ - private static class HiveTable { - - private final String name; - private final List<String> partitions; - - HiveTable(String name) { - this.name = name; - this.partitions = new ArrayList<>(); - } - - HiveTable add(String partition) { - partitions.add(partition); - return this; - } - } - - /** - * Representation of a Hive database. A database has a name and a list of tables - */ - private static class HiveDb { - - final String name; - Collection<HiveTable> tables; - - @SuppressWarnings("SameParameterValue") - HiveDb(String name) { - this.name = name; - tables = new ArrayList<>(); - } - - HiveDb(String name, Collection<HiveTable> tables) { - this.name = name; - this.tables = tables; - if (this.tables == null) { - this.tables = new ArrayList<>(); - } - } - - void add(HiveTable table) { - this.tables.add(table); - } - } - - /** - * Representation of a full Hive snapshot. A snapshot is collection of databases - */ - private static class HiveSnapshot { - - final List<HiveDb> databases = new ArrayList<>(); - - HiveSnapshot() { - } - - HiveSnapshot(Collection<HiveDb> dblist) { - if (dblist != null) { - databases.addAll(dblist); - } - } - - HiveSnapshot add(HiveDb db) { - this.databases.add(db); - return this; - } - } - - /** - * Mock for HMSClientFactory - */ - private static class MockHMSClientFactory implements HiveConnectionFactory { - - private HiveMetaStoreClient mClient; - - public MockHMSClientFactory() { - mClient = null; - } - - void setClient(MockClient mockClient) { - this.mClient = mockClient.client; - } - @Override - public HMSClient connect() throws IOException, InterruptedException, MetaException { - return new HMSClient(mClient); - } - - @Override - public void close() throws Exception { - } - } - - /** - * Convert Hive snapshot to mock client that will return proper values - * for the snapshot. - */ - private static class MockClient { - - public HiveMetaStoreClient client; - public long eventId; - - MockClient(HiveSnapshot snapshot, long eventId) throws TException { - this.eventId = eventId; - client = Mockito.mock(HiveMetaStoreClient.class); - List<String> dbNames = new ArrayList<>(snapshot.databases.size()); - // Walk over all databases and mock appropriate objects - for (HiveDb mdb : snapshot.databases) { - String dbName = mdb.name; - dbNames.add(dbName); - Database db = makeDb(dbName); - Mockito.when(client.getDatabase(dbName)).thenReturn(db); - List<String> tableNames = new ArrayList<>(mdb.tables.size()); - // Walk over all tables for the database and mock appropriate objects - for (HiveTable table : mdb.tables) { - String tableName = table.name; - tableNames.add(tableName); - Table mockTable = makeTable(dbName, tableName); - Mockito.when(client.getTableObjectsByName(dbName, - Lists.newArrayList(tableName))) - .thenReturn(Lists.newArrayList(mockTable)); - Mockito.when(client.listPartitionNames(dbName, tableName, (short) -1)) - .thenReturn(table.partitions); - // Walk across all partitions and mock appropriate objects - for (String partName : table.partitions) { - Partition p = makePartition(dbName, tableName, partName); - Mockito.when(client.getPartitionsByNames(dbName, tableName, - Lists.<String>newArrayList(partName))) - .thenReturn(Lists.<Partition>newArrayList(p)); - } - } - Mockito.when(client.getAllTables(dbName)).thenReturn(tableNames); - } - // Return all database names - Mockito.when(client.getAllDatabases()).thenReturn(dbNames); - Mockito.when(client.getCurrentNotificationEventId()). - thenReturn(new CurrentNotificationEventId(eventId)); - - } - - public Long incrementNotificationEventId() { - eventId = eventId + 1; - return eventId; - } - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/c56f48cc/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java index d619623..b9330cc 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java @@ -31,7 +31,7 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; -import org.apache.sentry.service.thrift.HmsFollower; +import org.apache.sentry.service.thrift.HMSFollower; import org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration; import org.junit.BeforeClass; import org.junit.Before; @@ -78,7 +78,7 @@ public class TestDbPrivilegeCleanupOnDrop extends to.close(); // Check the HMS connection only when notification log is enabled. if (enableNotificationLog) { - while (!HmsFollower.isConnectedToHms()) { + while (!HMSFollower.isConnectedToHMS()) { Thread.sleep(1000); } }
