Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign 50a127bf3 -> e5bb466ef
http://git-wip-us.apache.org/repos/asf/sentry/blob/e5bb466e/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java new file mode 100644 index 0000000..c6c9448 --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestNotificationProcessor.java @@ -0,0 +1,465 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.service.thrift; + +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hive.hcatalog.messaging.HCatEventMessage; +import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory; +import org.apache.sentry.hdfs.Updateable; +import org.apache.sentry.provider.db.service.persistent.SentryStore; +import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; + +// TODO 1. More tests should be added here. +// TODO 2. Tests using actual sentry store where. +@SuppressWarnings("unused") +public class TestNotificationProcessor { + + private static final SentryStore sentryStore = Mockito.mock(SentryStore.class); + private final static String hiveInstance = "server2"; + private final static Configuration conf = new Configuration(); + private final SentryJSONMessageFactory messageFactory = new SentryJSONMessageFactory(); + private NotificationProcessor notificationProcessor; + + @BeforeClass + public static void setup() { + conf.set("sentry.hive.sync.create", "true"); + conf.set("sentry.hive.sync.drop", "true"); + } + + @After + public void resetConf() { + conf.set("sentry.hive.sync.create", "true"); + conf.set("sentry.hive.sync.drop", "true"); + reset(sentryStore); + } + + @Test + /* + Makes sure that appropriate sentry store methods are invoked when create database event is + processed. + + Also, checks the hive sync configuration. + */ + public void testCreateDatabase() throws Exception { + long seqNum = 1; + String dbName = "db1"; + String uriPrefix = "hdfs:///"; + String location = "user/hive/warehouse"; + NotificationEvent notificationEvent; + TSentryAuthorizable authorizable; + notificationProcessor = new NotificationProcessor(sentryStore, + hiveInstance, conf); + + // Create notification event + notificationEvent = new NotificationEvent(seqNum, 0, + HCatEventMessage.EventType.CREATE_DATABASE.toString(), + messageFactory.buildCreateDatabaseMessage(new Database(dbName, + null, uriPrefix + location, null)).toString()); + + notificationProcessor.processNotificationEvent(notificationEvent); + + authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + authorizable.setDb("db1"); + //noinspection unchecked + verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), + Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + + verify(sentryStore, times(1)).dropPrivilege(authorizable, + NotificationProcessor.getPermUpdatableOnDrop(authorizable)); + reset(sentryStore); + + //Change the configuration and make sure that exiting privileges are not dropped + notificationProcessor.setSyncStoreOnCreate(false); + dbName = "db2"; + notificationEvent = new NotificationEvent(1, 0, + HCatEventMessage.EventType.CREATE_DATABASE.toString(), + messageFactory.buildCreateDatabaseMessage(new Database(dbName, + null, "hdfs:///db2", null)).toString()); + + notificationProcessor.processNotificationEvent(notificationEvent); + + authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + authorizable.setDb(dbName); + + //noinspection unchecked + verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), + Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + //making sure that privileges are not dropped + verify(sentryStore, times(0)).dropPrivilege(authorizable, + NotificationProcessor.getPermUpdatableOnDrop(authorizable)); + + } + + @Test + /* + Makes sure that appropriate sentry store methods are invoked when drop database event is + processed. + + Also, checks the hive sync configuration. + */ + public void testDropDatabase() throws Exception { + String dbName = "db1"; + + notificationProcessor = new NotificationProcessor(sentryStore, + hiveInstance, conf); + + // Create notification event + NotificationEvent notificationEvent = new NotificationEvent(1, 0, + HCatEventMessage.EventType.DROP_DATABASE.toString(), + messageFactory.buildDropDatabaseMessage(new Database(dbName, null, + "hdfs:///db1", null)).toString()); + + notificationProcessor.processNotificationEvent(notificationEvent); + + TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + authorizable.setDb("db1"); + + //noinspection unchecked + verify(sentryStore, times(1)).deleteAuthzPathsMapping(Mockito.anyString(), + Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + verify(sentryStore, times(1)).dropPrivilege(authorizable, + NotificationProcessor.getPermUpdatableOnDrop(authorizable)); + reset(sentryStore); + + // Change the configuration and make sure that exiting privileges are not dropped + notificationProcessor.setSyncStoreOnDrop(false); + dbName = "db2"; + // Create notification event + notificationEvent = new NotificationEvent(1, 0, + HCatEventMessage.EventType.DROP_DATABASE.toString(), + messageFactory.buildDropDatabaseMessage(new Database(dbName, null, + "hdfs:///db2", null)).toString()); + + notificationProcessor.processNotificationEvent(notificationEvent); + + authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + authorizable.setDb(dbName); + + //noinspection unchecked + verify(sentryStore, times(1)).deleteAuthzPathsMapping(Mockito.anyString(), + Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + verify(sentryStore, times(0)).dropPrivilege(authorizable, + NotificationProcessor.getPermUpdatableOnDrop(authorizable)); + } + + @Test + /* + Makes sure that appropriate sentry store methods are invoked when create table event is + processed. + + Also, checks the hive sync configuration. + */ + public void testCreateTable() throws Exception { + String dbName = "db1"; + String tableName = "table1"; + + notificationProcessor = new NotificationProcessor(sentryStore, + hiveInstance, conf); + + // Create notification event + StorageDescriptor sd = new StorageDescriptor(); + sd.setLocation("hdfs:///db1.db/table1"); + NotificationEvent notificationEvent = + new NotificationEvent(1, 0, HCatEventMessage.EventType.CREATE_TABLE.toString(), + messageFactory.buildCreateTableMessage(new Table(tableName, + dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString()); + + notificationProcessor.processNotificationEvent(notificationEvent); + + TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + authorizable.setDb("db1"); + authorizable.setTable(tableName); + + //noinspection unchecked + verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), + Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + + verify(sentryStore, times(1)).dropPrivilege(authorizable, + NotificationProcessor.getPermUpdatableOnDrop(authorizable)); + reset(sentryStore); + + // Change the configuration and make sure that existing privileges are not dropped + notificationProcessor.setSyncStoreOnCreate(false); + + // Create notification event + dbName = "db2"; + tableName = "table2"; + sd = new StorageDescriptor(); + sd.setLocation("hdfs:///db1.db/table2"); + notificationEvent = + new NotificationEvent(1, 0, HCatEventMessage.EventType.CREATE_TABLE.toString(), + messageFactory.buildCreateTableMessage(new Table(tableName, + dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString()); + + notificationProcessor.processNotificationEvent(notificationEvent); + + authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + authorizable.setDb(dbName); + authorizable.setTable(tableName); + + //noinspection unchecked + verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), + Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + // Making sure that privileges are not dropped + verify(sentryStore, times(0)).dropPrivilege(authorizable, + NotificationProcessor.getPermUpdatableOnDrop(authorizable)); + } + + @Test + /* + Makes sure that appropriate sentry store methods are invoked when drop table event is + processed. + + Also, checks the hive sync configuration. + */ + public void testDropTable() throws Exception { + String dbName = "db1"; + String tableName = "table1"; + + Configuration authConf = new Configuration(); + notificationProcessor = new NotificationProcessor(sentryStore, + hiveInstance, authConf); + + // Create notification event + StorageDescriptor sd = new StorageDescriptor(); + sd.setLocation("hdfs:///db1.db/table1"); + NotificationEvent notificationEvent = new NotificationEvent(1, 0, + HCatEventMessage.EventType.DROP_TABLE.toString(), + messageFactory.buildDropTableMessage(new Table(tableName, + dbName, null, 0, 0, 0, sd, null, null, null, null, null)).toString()); + + notificationProcessor.processNotificationEvent(notificationEvent); + + TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + authorizable.setDb("db1"); + authorizable.setTable(tableName); + + verify(sentryStore, times(1)).deleteAllAuthzPathsMapping(Mockito.anyString(), + Mockito.any(Updateable.Update.class)); + + verify(sentryStore, times(1)).dropPrivilege(authorizable, + NotificationProcessor.getPermUpdatableOnDrop(authorizable)); + } + + @Test + /* + Makes sure that appropriate sentry store methods are invoked when alter tables event is + processed. + */ + public void testAlterTable() throws Exception { + String dbName = "db1"; + String tableName = "table1"; + + String newDbName = "db1"; + String newTableName = "table2"; + + Configuration authConf = new Configuration(); + notificationProcessor = new NotificationProcessor(sentryStore, + hiveInstance, authConf); + + // Create notification event + StorageDescriptor sd = new StorageDescriptor(); + sd.setLocation("hdfs:///db1.db/table1"); + NotificationEvent notificationEvent = new NotificationEvent(1, 0, + HCatEventMessage.EventType.ALTER_TABLE.toString(), + messageFactory.buildAlterTableMessage( + new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null), + new Table(newTableName, newDbName, null, 0, 0, 0, sd, null, null, null, null, null)) + .toString()); + notificationEvent.setDbName(newDbName); + notificationEvent.setTableName(newTableName); + + notificationProcessor.processNotificationEvent(notificationEvent); + + TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + authorizable.setDb(dbName); + authorizable.setTable(tableName); + + TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + newAuthorizable.setDb(newDbName); + newAuthorizable.setTable(newTableName); + + verify(sentryStore, times(1)).renameAuthzObj(Mockito.anyString(), Mockito.anyString(), + Mockito.any(Updateable.Update.class)); + + verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable, + NotificationProcessor.getPermUpdatableOnRename(authorizable, newAuthorizable)); + } + + @Test + /* + Makes sure that appropriate sentry store methods are invoked when alter tables event is + processed. + */ + public void testRenameTableWithLocationUpdate() throws Exception { + String dbName = "db1"; + String tableName = "table1"; + + String newDbName = "db1"; + String newTableName = "table2"; + + Configuration authConf = new Configuration(); + notificationProcessor = new NotificationProcessor(sentryStore, + hiveInstance, authConf); + + // Create notification event + StorageDescriptor sd = new StorageDescriptor(); + sd.setLocation("hdfs:///db1.db/table1"); + StorageDescriptor new_sd = new StorageDescriptor(); + new_sd.setLocation("hdfs:///db1.db/table2"); + NotificationEvent notificationEvent = new NotificationEvent(1, 0, + HCatEventMessage.EventType.ALTER_TABLE.toString(), + messageFactory.buildAlterTableMessage( + new Table(tableName, dbName, null, 0, 0, 0, sd, null, null, null, null, null), + new Table(newTableName, newDbName, null, 0, 0, 0, new_sd, null, null, null, null, null)) + .toString()); + notificationEvent.setDbName(newDbName); + notificationEvent.setTableName(newTableName); + + notificationProcessor.processNotificationEvent(notificationEvent); + + TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + authorizable.setDb(dbName); + authorizable.setTable(tableName); + + TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance); + authorizable.setServer(hiveInstance); + newAuthorizable.setDb(newDbName); + newAuthorizable.setTable(newTableName); + + verify(sentryStore, times(1)).renameAuthzPathsMapping(Mockito.anyString(), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), Mockito.any(Updateable.Update.class)); + + verify(sentryStore, times(1)).renamePrivilege(authorizable, newAuthorizable, + NotificationProcessor.getPermUpdatableOnRename(authorizable, newAuthorizable)); + } + + @Test + /* + Test to made sure that sentry store is not invoked when invalid alter table event is + processed. + */ + public void testAlterTableWithInvalidEvent() throws Exception { + String dbName = "db1"; + String tableName1 = "table1"; + String tableName2 = "table2"; + long inputEventId = 1; + NotificationEvent notificationEvent; + List<FieldSchema> partCols; + StorageDescriptor sd; + Mockito.doNothing().when(sentryStore).persistLastProcessedNotificationID(Mockito.anyLong()); + //noinspection unchecked + Mockito.doNothing().when(sentryStore).addAuthzPathsMapping(Mockito.anyString(), + Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + + Configuration authConf = new Configuration(); + notificationProcessor = new NotificationProcessor(sentryStore, + hiveInstance, authConf); + + // Create a table + sd = new StorageDescriptor(); + sd.setLocation("hdfs://db1.db/table1"); + partCols = new ArrayList<>(); + partCols.add(new FieldSchema("ds", "string", "")); + Table table = new Table(tableName1, dbName, null, 0, 0, 0, sd, partCols, + null, null, null, null); + notificationEvent = new NotificationEvent(inputEventId, 0, + HCatEventMessage.EventType.CREATE_TABLE.toString(), + messageFactory.buildCreateTableMessage(table).toString()); + notificationEvent.setDbName(dbName); + notificationEvent.setTableName(tableName1); + inputEventId += 1; + // Process the notification + notificationProcessor.processNotificationEvent(notificationEvent); + // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification + // and persistLastProcessedNotificationID was not invoked. + //noinspection unchecked + verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), + Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + reset(sentryStore); + + // Create alter table notification with out actually changing anything. + // This notification should not be processed by sentry server + // Notification should be persisted explicitly + notificationEvent = new NotificationEvent(1, 0, + HCatEventMessage.EventType.ALTER_TABLE.toString(), + messageFactory.buildAlterTableMessage( + new Table(tableName1, dbName, null, 0, 0, 0, sd, null, null, null, null, null), + new Table(tableName1, dbName, null, 0, 0, 0, sd, null, + null, null, null, null)).toString()); + notificationEvent.setDbName(dbName); + notificationEvent.setTableName(tableName1); + inputEventId += 1; + // Process the notification + notificationProcessor.processNotificationEvent(notificationEvent); + // Make sure that renameAuthzObj and deleteAuthzPathsMapping were not invoked + // to handle CREATE_TABLE notification + // and persistLastProcessedNotificationID is explicitly invoked + verify(sentryStore, times(0)).renameAuthzObj(Mockito.anyString(), Mockito.anyString(), + Mockito.any(Updateable.Update.class)); + //noinspection unchecked + verify(sentryStore, times(0)).deleteAuthzPathsMapping(Mockito.anyString(), + Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + reset(sentryStore); + + // Create a table + sd = new StorageDescriptor(); + sd.setLocation("hdfs://db1.db/table2"); + partCols = new ArrayList<>(); + partCols.add(new FieldSchema("ds", "string", "")); + Table table1 = new Table(tableName2, dbName, null, 0, 0, 0, sd, + partCols, null, null, null, null); + notificationEvent = new NotificationEvent(inputEventId, 0, + HCatEventMessage.EventType.CREATE_TABLE.toString(), + messageFactory.buildCreateTableMessage(table1).toString()); + notificationEvent.setDbName(dbName); + notificationEvent.setTableName(tableName2); + // Process the notification + notificationProcessor.processNotificationEvent(notificationEvent); + // Make sure that addAuthzPathsMapping was invoked once to handle CREATE_TABLE notification + // and persistLastProcessedNotificationID was not invoked. + //noinspection unchecked + verify(sentryStore, times(1)).addAuthzPathsMapping(Mockito.anyString(), + Mockito.anyCollection(), Mockito.any(Updateable.Update.class)); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/e5bb466e/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java new file mode 100644 index 0000000..3cc6541 --- /dev/null +++ b/sentry-provider/sentry-provider-db/src/test/java/org/apache/sentry/service/thrift/TestSentryHmsClient.java @@ -0,0 +1,470 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.service.thrift; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Random; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.IMetaStoreClient.NotificationFilter; +import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hive.hcatalog.messaging.HCatEventMessage; +import org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory; +import org.apache.sentry.provider.db.service.persistent.PathsImage; +import org.apache.thrift.TException; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import javax.security.auth.login.LoginException; + +/** + * Test mocks HiveMetaStoreClient class and tests SentryHmsClient. + */ +public class TestSentryHmsClient { + + private static final Configuration conf = new Configuration(); + private static final SentryJSONMessageFactory messageFactory = new SentryJSONMessageFactory(); + private static SentryHmsClient client; + private static MockHMSClientFactory hiveConnectionFactory; + + /** + * Create mock database with the given name + * + * @param name Database name + * @return Mock database object + */ + private static Database makeDb(String name) { + Database db = Mockito.mock(Database.class); + Mockito.when(db.getName()).thenReturn(name); + Mockito.when(db.getLocationUri()).thenReturn("hdfs:///" + name); + return db; + } + + /** + * Create mock table + * + * @param dbName db for this table + * @param tableName name of the table + * @return mock table object + */ + private static Table makeTable(String dbName, String tableName) { + Table table = Mockito.mock(Table.class); + Mockito.when(table.getDbName()).thenReturn(dbName); + Mockito.when(table.getTableName()).thenReturn(tableName); + StorageDescriptor sd = Mockito.mock(StorageDescriptor.class); + Mockito.when(sd.getLocation()).thenReturn( + String.format("hdfs:///%s/%s", dbName, tableName)); + Mockito.when(table.getSd()).thenReturn(sd); + return table; + } + + /** + * Create mock partition + * + * @param dbName database for this partition + * @param tableName table for this partition + * @param partName partition name + * @return mock partition object + */ + private static Partition makePartition(String dbName, String tableName, String partName) { + Partition partition = Mockito.mock(Partition.class); + StorageDescriptor sd = Mockito.mock(StorageDescriptor.class); + Mockito.when(sd.getLocation()).thenReturn( + String.format("hdfs:///%s/%s/%s", dbName, tableName, partName)); + Mockito.when(partition.getSd()).thenReturn(sd); + return partition; + } + + /** + * Creates create database notification + * + * @return NotificationEvent + */ + private static NotificationEvent getCreateDatabaseNotification(long id) { + Random rand = new Random(); + int n = rand.nextInt(100) + 1; + String dbName = "db" + n; + return new NotificationEvent(id, 0, HCatEventMessage.EventType.CREATE_DATABASE.toString(), + messageFactory + .buildCreateDatabaseMessage(new Database(dbName, null, "hdfs:///" + dbName, null)) + .toString()); + } + + /** + * Creates drop database notification + * + * @return NotificationEvent + */ + private static NotificationEvent getDropDatabaseNotification(long id) { + Random rand = new Random(); + int n = rand.nextInt(100) + 1; + String dbName = "db" + n; + return new NotificationEvent(id, 0, HCatEventMessage.EventType.DROP_DATABASE.toString(), + messageFactory + .buildDropDatabaseMessage(new Database(dbName, null, "hdfs:///" + dbName, null)) + .toString()); + } + + @BeforeClass + static public void initialize() throws IOException, LoginException { + hiveConnectionFactory = new MockHMSClientFactory(); + client = new SentryHmsClient(conf, (HiveConnectionFactory)hiveConnectionFactory); + } + + /** + * Creating snapshot when SentryHmsClient is not connected to HMS + */ + @Test + public void testSnapshotCreationWithOutClientConnected() throws Exception { + // Make sure that client is not connected + Assert.assertFalse(client.isConnected()); + PathsImage snapshotInfo = client.getFullSnapshot(); + Assert.assertTrue(snapshotInfo.getPathImage().isEmpty()); + } + + /** + * Creating snapshot when HMS doesn't have any data + */ + @Test + public void testSnapshotCreationWithNoHmsData() throws Exception { + MockClient mockClient = new MockClient(new HiveSnapshot(), 1); + client.setClient(mockClient.client); + // Make sure that client is connected + Assert.assertTrue(client.isConnected()); + PathsImage snapshotInfo = client.getFullSnapshot(); + Assert.assertTrue(snapshotInfo.getPathImage().isEmpty()); + } + + /** + * Creating a snapshot when there is data but there are updates to HMS data mean while + */ + @Test + public void testSnapshotCreationWhenDataIsActivelyUpdated() throws Exception { + HiveTable tab21 = new HiveTable("tab21"); + HiveTable tab31 = new HiveTable("tab31").add("part311").add("part312"); + HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31)); + HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21)); + HiveDb db1 = new HiveDb("db1"); + HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3); + final MockClient mockClient = new MockClient(snap, 1); + + client.setClient(mockClient.client); + hiveConnectionFactory.setClient(mockClient); + // Make sure that client is connected + Assert.assertTrue(client.isConnected()); + PathsImage snapshotInfo = client.getFullSnapshot(); + // Make sure that snapshot is not empty + Assert.assertTrue(!snapshotInfo.getPathImage().isEmpty()); + + Mockito.when(mockClient.client.getCurrentNotificationEventId()). + thenAnswer(new Answer<CurrentNotificationEventId>() { + @Override + public CurrentNotificationEventId answer(InvocationOnMock invocation) + throws Throwable { + return new CurrentNotificationEventId(mockClient.incrementNotificationEventId()); + } + + }); + + snapshotInfo = client.getFullSnapshot(); + Assert.assertTrue(snapshotInfo.getPathImage().isEmpty()); + } + + /** + * Creating a snapshot when there is data in HMS. + */ + @Test + public void testSnapshotCreationSuccess() throws Exception { + HiveTable tab21 = new HiveTable("tab21"); + HiveTable tab31 = new HiveTable("tab31"); + HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31)); + HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21)); + HiveDb db1 = new HiveDb("db1"); + HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3); + MockClient mockClient = new MockClient(snap, 1); + Mockito.when(mockClient.client.getCurrentNotificationEventId()). + thenReturn(new CurrentNotificationEventId(mockClient.eventId)); + client.setClient(mockClient.client); + hiveConnectionFactory.setClient(mockClient); + // Make sure that client is connected + Assert.assertTrue(client.isConnected()); + + PathsImage snapshotInfo = client.getFullSnapshot(); + Assert.assertEquals(5, snapshotInfo.getPathImage().size()); + Assert.assertEquals(Sets.newHashSet("db1"), snapshotInfo.getPathImage().get("db1")); + Assert.assertEquals(Sets.newHashSet("db2"), snapshotInfo.getPathImage().get("db2")); + Assert.assertEquals(Sets.newHashSet("db3"), snapshotInfo.getPathImage().get("db3")); + Assert.assertEquals(Sets.newHashSet("db2/tab21"), + snapshotInfo.getPathImage().get("db2.tab21")); + Assert.assertEquals(Sets.newHashSet("db3/tab31"), snapshotInfo.getPathImage().get("db3.tab31")); + Assert.assertEquals(snapshotInfo.getId(), mockClient.eventId); + + } + + /** + * Test scenario when there is no HMS connection + * Getting new notifications + */ + @Test + public void testGetNewNotificationsWithOutClientConnected() throws Exception { + HiveTable tab21 = new HiveTable("tab21"); + HiveTable tab31 = new HiveTable("tab31"); + HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31)); + HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21)); + HiveDb db1 = new HiveDb("db1"); + client.setClient(null); + HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3); + MockClient mockClient = new MockClient(snap, 100); + Mockito.when(mockClient.client.getCurrentNotificationEventId()). + thenReturn(new CurrentNotificationEventId(mockClient.eventId)); + // Make sure that client is not connected + Assert.assertTrue(!client.isConnected()); + Collection<NotificationEvent> events = client.getNotifications(100); + Assert.assertTrue(events.isEmpty()); + + } + + /** + * Test scenario where there are no notifications + * Getting new notifications + */ + @Test + public void testGetNewNotificationsWithNoHmsUpdates() throws Exception { + HiveTable tab21 = new HiveTable("tab21"); + HiveTable tab31 = new HiveTable("tab31"); + HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31)); + HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21)); + HiveDb db1 = new HiveDb("db1"); + HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3); + MockClient mockClient = new MockClient(snap, 100); + Mockito.when(mockClient.client.getCurrentNotificationEventId()). + thenReturn(new CurrentNotificationEventId(mockClient.eventId)); + client.setClient(mockClient.client); + hiveConnectionFactory.setClient(mockClient); + // Make sure that client is connected + Assert.assertTrue(client.isConnected()); + Collection<NotificationEvent> events = client.getNotifications(100); + Assert.assertTrue(events.isEmpty()); + } + + /** + * Test scenario where there are notifications + * Getting new notifications + */ + @Test + public void testGetNewNotificationsSuccess() throws Exception { + final MockClient mockClient = new MockClient(new HiveSnapshot(), 100); + client.setClient(mockClient.client); + hiveConnectionFactory.setClient(mockClient); + // Make sure that client is connected + Assert.assertTrue(client.isConnected()); + + Mockito.when(mockClient.client.getCurrentNotificationEventId()). + thenAnswer(new Answer<CurrentNotificationEventId>() { + @Override + public CurrentNotificationEventId answer(InvocationOnMock invocation) + throws Throwable { + return new CurrentNotificationEventId(mockClient.incrementNotificationEventId()); + } + }); + Mockito.when(mockClient.client.getNextNotification(Mockito.anyLong(), Mockito.anyInt(), + Mockito.any(NotificationFilter.class))). + thenAnswer(new Answer<NotificationEventResponse>() { + @Override + public NotificationEventResponse answer(InvocationOnMock invocation) + throws Throwable { + long id = 1; + List<NotificationEvent> events = new ArrayList<>(); + events.add(getCreateDatabaseNotification(id++)); + events.add(getDropDatabaseNotification(id++)); + return new NotificationEventResponse(events); + } + }); + + Collection<NotificationEvent> events = client.getNotifications(100); + long id = 1; + for (NotificationEvent event : events) { + Assert.assertEquals(event.getEventId(), id++); + } + Assert.assertTrue(events.size() == 2); + } + + /** + * Representation of a Hive table. A table has a name and a list of partitions. + */ + private static class HiveTable { + + private final String name; + private final List<String> partitions; + + HiveTable(String name) { + this.name = name; + this.partitions = new ArrayList<>(); + } + + HiveTable add(String partition) { + partitions.add(partition); + return this; + } + } + + /** + * Representation of a Hive database. A database has a name and a list of tables + */ + private static class HiveDb { + + final String name; + Collection<HiveTable> tables; + + @SuppressWarnings("SameParameterValue") + HiveDb(String name) { + this.name = name; + tables = new ArrayList<>(); + } + + HiveDb(String name, Collection<HiveTable> tables) { + this.name = name; + this.tables = tables; + if (this.tables == null) { + this.tables = new ArrayList<>(); + } + } + + void add(HiveTable table) { + this.tables.add(table); + } + } + + /** + * Representation of a full Hive snapshot. A snapshot is collection of databases + */ + private static class HiveSnapshot { + + final List<HiveDb> databases = new ArrayList<>(); + + HiveSnapshot() { + } + + HiveSnapshot(Collection<HiveDb> dblist) { + if (dblist != null) { + databases.addAll(dblist); + } + } + + HiveSnapshot add(HiveDb db) { + this.databases.add(db); + return this; + } + } + + /** + * Mock for HMSClientFactory + */ + private static class MockHMSClientFactory implements HiveConnectionFactory { + + private HiveMetaStoreClient mClient; + + public MockHMSClientFactory() { + mClient = null; + } + + void setClient(MockClient mockClient) { + this.mClient = mockClient.client; + } + @Override + public HMSClient connect() throws IOException, InterruptedException, MetaException { + return new HMSClient(mClient); + } + + @Override + public void close() throws Exception { + } + } + + /** + * Convert Hive snapshot to mock client that will return proper values + * for the snapshot. + */ + private static class MockClient { + + public HiveMetaStoreClient client; + public long eventId; + + MockClient(HiveSnapshot snapshot, long eventId) throws TException { + this.eventId = eventId; + client = Mockito.mock(HiveMetaStoreClient.class); + List<String> dbNames = new ArrayList<>(snapshot.databases.size()); + // Walk over all databases and mock appropriate objects + for (HiveDb mdb : snapshot.databases) { + String dbName = mdb.name; + dbNames.add(dbName); + Database db = makeDb(dbName); + Mockito.when(client.getDatabase(dbName)).thenReturn(db); + List<String> tableNames = new ArrayList<>(mdb.tables.size()); + // Walk over all tables for the database and mock appropriate objects + for (HiveTable table : mdb.tables) { + String tableName = table.name; + tableNames.add(tableName); + Table mockTable = makeTable(dbName, tableName); + Mockito.when(client.getTableObjectsByName(dbName, + Lists.newArrayList(tableName))) + .thenReturn(Lists.newArrayList(mockTable)); + Mockito.when(client.listPartitionNames(dbName, tableName, (short) -1)) + .thenReturn(table.partitions); + // Walk across all partitions and mock appropriate objects + for (String partName : table.partitions) { + Partition p = makePartition(dbName, tableName, partName); + Mockito.when(client.getPartitionsByNames(dbName, tableName, + Lists.<String>newArrayList(partName))) + .thenReturn(Lists.<Partition>newArrayList(p)); + } + } + Mockito.when(client.getAllTables(dbName)).thenReturn(tableNames); + } + // Return all database names + Mockito.when(client.getAllDatabases()).thenReturn(dbNames); + Mockito.when(client.getCurrentNotificationEventId()). + thenReturn(new CurrentNotificationEventId(eventId)); + + } + + public Long incrementNotificationEventId() { + eventId = eventId + 1; + return eventId; + } + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/e5bb466e/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java index b9330cc..d619623 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/dbprovider/TestDbPrivilegeCleanupOnDrop.java @@ -31,7 +31,7 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; -import org.apache.sentry.service.thrift.HMSFollower; +import org.apache.sentry.service.thrift.HmsFollower; import org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration; import org.junit.BeforeClass; import org.junit.Before; @@ -78,7 +78,7 @@ public class TestDbPrivilegeCleanupOnDrop extends to.close(); // Check the HMS connection only when notification log is enabled. if (enableNotificationLog) { - while (!HMSFollower.isConnectedToHMS()) { + while (!HmsFollower.isConnectedToHms()) { Thread.sleep(1000); } }
