Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign a70cff999 -> 73d77bcd0
http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java new file mode 100644 index 0000000..56e19c4 --- /dev/null +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDBNotificationListenerInBuiltDeserializer.java @@ -0,0 +1,353 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.tests.e2e.metastore; + +import com.google.common.collect.Lists; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.*; +import org.apache.hive.hcatalog.messaging.CreateDatabaseMessage; +import org.apache.hive.hcatalog.messaging.HCatEventMessage; +import org.apache.hive.hcatalog.messaging.MessageDeserializer; +import org.apache.hive.hcatalog.messaging.MessageFactory; +import org.apache.hive.hcatalog.messaging.CreateTableMessage; +import org.apache.hive.hcatalog.messaging.DropTableMessage; +import org.apache.hive.hcatalog.messaging.AlterTableMessage; +import org.apache.hive.hcatalog.messaging.AlterPartitionMessage; +import org.apache.hive.hcatalog.messaging.DropDatabaseMessage; +import org.apache.hive.hcatalog.messaging.AddPartitionMessage; +import org.apache.hive.hcatalog.messaging.DropPartitionMessage; +import org.apache.sentry.tests.e2e.hive.StaticUserGroup; +import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory; +import org.hamcrest.text.IsEqualIgnoringCase; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +import org.junit.*; + +import java.io.File; +import java.util.ArrayList; +import java.util.Random; + +/** + * Make sure NotificationLog is capturing the information correctly for the commands which change <Obj,Location> mapping + * This test class is using Hive's DbNotificationListener and Hive's Notification log JSON deserializer. + */ + +public class TestDBNotificationListenerInBuiltDeserializer extends AbstractMetastoreTestWithStaticConfiguration { + + protected static HiveMetaStoreClient client; + protected static MessageDeserializer deserializer; + protected static Random random = new Random(); + private static String testDB; + + @BeforeClass + public static void setupTestStaticConfiguration() throws Exception { + setMetastoreListener = true; + useDbNotificationListener = true; + beforeClass(); + } + + protected static void beforeClass() throws Exception { + AbstractMetastoreTestWithStaticConfiguration.setupTestStaticConfiguration(); + client = context.getMetaStoreClient(ADMIN1); + deserializer = MessageFactory.getDeserializer("json", ""); + writePolicyFile(setAdminOnServer1(ADMINGROUP).setUserGroupMapping(StaticUserGroup.getStaticMapping())); + } + + @AfterClass + public static void cleanupAfterClass() throws Exception { + if (client != null) { + client.close(); + } + } + + @After + public void dropDBAfterTest() throws Exception { + if(client != null && testDB != null) { + dropMetastoreDBIfExists(client, testDB); + } + } + + @Test + public void testCreateDropDatabase() throws Exception { + CurrentNotificationEventId latestID, previousID; + NotificationEventResponse response; + + testDB = "N_db" + random.nextInt(Integer.SIZE - 1); + + // Create database + // We need: + // - Dbname + // - location + createMetastoreDB(client, testDB); + latestID = client.getCurrentNotificationEventId(); + response = client.getNextNotification(latestID.getEventId() - 1, 1, null); + CreateDatabaseMessage createDatabaseMessage = deserializer.getCreateDatabaseMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.CREATE_DATABASE, createDatabaseMessage.getEventType()); //Validate EventType + assertEquals(testDB, createDatabaseMessage.getDB()); //dbName + //Location information is not available + + //Alter database location and rename are not supported. See HIVE-4847 + + //Drop database + // We need: + // - dbName + // - location + client.dropDatabase(testDB); + previousID = latestID; + latestID = client.getCurrentNotificationEventId(); + assertEquals(previousID.getEventId() + 1, latestID.getEventId()); //Validate monotonically increasing eventID + response = client.getNextNotification(latestID.getEventId() - 1, 1, null); + DropDatabaseMessage dropDatabaseMessage = deserializer.getDropDatabaseMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.DROP_DATABASE, dropDatabaseMessage.getEventType()); //Event type + assertThat(dropDatabaseMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB)); // dbName + //Location information is not available, but we might not really need it as we can drop all paths associated with + //the object when we drop + } + + @Test + public void testCreateDropTableWithPartition() throws Exception { + testDB = "N_db" + random.nextInt(Integer.SIZE - 1); + String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); + + NotificationEventResponse response; + CurrentNotificationEventId latestID, previousID; + // Create database + createMetastoreDB(client, testDB); + + // Create table with partition + // We need: + // - dbname + // - tablename + // - location + createMetastoreTableWithPartition(client, testDB, + testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")), + Lists.newArrayList(new FieldSchema("part_col1", "string", ""))); + latestID = client.getCurrentNotificationEventId(); + response = client.getNextNotification(latestID.getEventId() - 1, 1, null); + CreateTableMessage createTableMessage = deserializer.getCreateTableMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.CREATE_TABLE, createTableMessage.getEventType()); + assertEquals(testDB, createTableMessage.getDB()); //dbName + assertEquals(testTable, createTableMessage.getTable()); //tableName + //Location information is not available + + //Drop table + // We need: + // - dbName + // - tableName + // - location + client.dropTable(testDB, testTable); + previousID = latestID; + latestID = client.getCurrentNotificationEventId(); + assertEquals(previousID.getEventId() + 1, latestID.getEventId()); + response = client.getNextNotification(latestID.getEventId() - 1, 1, null); + DropTableMessage dropTableMessage = deserializer.getDropTableMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.DROP_TABLE, dropTableMessage.getEventType()); + assertThat(dropTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName + assertThat(dropTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName + //Location information is not available, but we might not really need it as we can drop all paths associated with + //the object when we drop + } + + @Test + public void testCreateDropTableWithoutPartition() throws Exception { + testDB = "N_db" + random.nextInt(Integer.SIZE - 1); + String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); + + NotificationEventResponse response; + CurrentNotificationEventId latestID, previousID; + // Create database + createMetastoreDB(client, testDB); + + // Create table with partition + // We need: + // - dbname + // - tablename + // - location + createMetastoreTable(client, testDB, testTable, Lists.newArrayList(new FieldSchema("col1", "int", ""))); + latestID = client.getCurrentNotificationEventId(); + response = client.getNextNotification(latestID.getEventId() - 1, 1, null); + CreateTableMessage createTableMessage = deserializer.getCreateTableMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.CREATE_TABLE, createTableMessage.getEventType()); + assertEquals(testDB, createTableMessage.getDB()); //dbName + assertEquals(testTable, createTableMessage.getTable()); //tableName + //Location information is not available + + //Drop table + // We need: + // - dbName + // - tableName + // - location + client.dropTable(testDB, testTable); + previousID = latestID; + latestID = client.getCurrentNotificationEventId(); + assertEquals(previousID.getEventId() + 1, latestID.getEventId()); + response = client.getNextNotification(latestID.getEventId() - 1, 1, null); + DropTableMessage dropTableMessage = deserializer.getDropTableMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.DROP_TABLE, dropTableMessage.getEventType()); + assertThat(dropTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName + assertThat(dropTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName + //Location information is not available, but we might not really need it as we can drop all paths associated with + //the object when we drop + } + + @Test + public void testAddDropPartition() throws Exception { + testDB = "N_db" + random.nextInt(Integer.SIZE - 1); + String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); + + NotificationEventResponse response; + CurrentNotificationEventId latestID, previousID; + // Create database and table + createMetastoreDB(client, testDB); + Table tbl1 = createMetastoreTableWithPartition(client, testDB, testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")), + Lists.newArrayList(new FieldSchema("part_col1", "string", ""))); + + ArrayList<String> partVals1 = Lists.newArrayList("part1"); + + //Add partition + // We need: + // - dbName + // - tableName + // - partition location + addPartition(client, testDB, testTable, partVals1, tbl1); + latestID = client.getCurrentNotificationEventId(); + response = client.getNextNotification(latestID.getEventId() - 1, 1, null); + AddPartitionMessage addPartitionMessage = deserializer.getAddPartitionMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.ADD_PARTITION, addPartitionMessage.getEventType()); + assertThat(addPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));// dbName (returns lowered version) + assertThat(addPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));// tableName (returns lowered version) + //Location information is not available + + //Drop partition + // We need: + // - dbName + // - tableName + // - partition location + dropPartition(client, testDB, testTable, partVals1); + previousID = latestID; + latestID = client.getCurrentNotificationEventId(); + assertEquals(previousID.getEventId() + 1, latestID.getEventId()); + response = client.getNextNotification(latestID.getEventId() - 1, 1, null); + DropPartitionMessage dropPartitionMessage = deserializer.getDropPartitionMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.DROP_PARTITION, dropPartitionMessage.getEventType()); + assertThat(dropPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB)); //dbName + assertThat(dropPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable)); //tableName + //Location information is not available + + } + + @Ignore("Needs Hive >= 1.1.2") + @Test + public void testAlterTableWithPartition() throws Exception { + testDB = "N_db" + random.nextInt(Integer.SIZE - 1); + String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); + + NotificationEventResponse response; + CurrentNotificationEventId latestID, previousID; + // Create database + createMetastoreDB(client, testDB); + + // Create table with partition + Table tbl1 = createMetastoreTableWithPartition(client, testDB, + testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")), + Lists.newArrayList(new FieldSchema("part_col1", "string", ""))); + + //Alter table location + // We need: + // - dbName + // - tableName + // - old location + // - new location + String tabDir1 = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR) + + File.separator + random.nextInt(Integer.SIZE - 1); + alterTableWithLocation(client, tbl1, tabDir1); + latestID = client.getCurrentNotificationEventId(); + response = client.getNextNotification(latestID.getEventId()-1, 1, null); + AlterTableMessage alterTableMessage = deserializer.getAlterTableMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.ALTER_TABLE, alterTableMessage.getEventType()); + assertThat(alterTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName + assertThat(alterTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName + //Old location is not available: This information is lost if not captured at the time of event. + //New location is not available + + //Alter table rename managed table - location also changes + // We need: + // - oldDbName + // - newDbName + // - oldTableName + // - newTableName + // - old location + // - new location + String newDBName = testDB + random.nextInt(Integer.SIZE - 1); + String newTableName = testTable + random.nextInt(Integer.SIZE - 1); + String newLocation = tabDir1 + random.nextInt(Integer.SIZE - 1); + createMetastoreDB(client, newDBName); + alterTableRename(client, tbl1, newDBName, newTableName, newLocation); + previousID = latestID; + latestID = client.getCurrentNotificationEventId(); + assertEquals(previousID.getEventId() + 1, latestID.getEventId()); + response = client.getNextNotification(latestID.getEventId()-1, 1, null); + alterTableMessage = deserializer.getAlterTableMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.ALTER_TABLE, alterTableMessage.getEventType()); + assertThat(alterTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//oldDbName + assertThat(alterTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//oldTableName + assertThat(response.getEvents().get(0).getDbName(), IsEqualIgnoringCase.equalToIgnoringCase(newDBName));//newDbName + assertThat(response.getEvents().get(0).getTableName(), IsEqualIgnoringCase.equalToIgnoringCase(newTableName));//newTableName + //Old location: This information is lost if not captured at the time of event. + //New location: Not sure how can we get this? Refresh all paths for every alter table add partition? + } + + @Ignore("Needs Hive >= 1.1.2") + @Test + public void testAlterPartition() throws Exception { + testDB = "N_db" + random.nextInt(Integer.SIZE - 1); + String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); + + NotificationEventResponse response; + CurrentNotificationEventId latestID; + // Create database + createMetastoreDB(client, testDB); + + // Create table with partition + Table tbl1 = createMetastoreTableWithPartition(client, testDB, + testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")), + Lists.newArrayList(new FieldSchema("part_col1", "string", ""))); + ArrayList<String> partVals1 = Lists.newArrayList("part1"); + Partition partition = addPartition(client, testDB, testTable, partVals1, tbl1); + + + String warehouseDir = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR); + //Alter partition with location + // We need: + // - dbName + // - tableName + // - partition location + alterPartitionWithLocation(client, partition, warehouseDir + File.separator + "newpart"); + latestID = client.getCurrentNotificationEventId(); + response = client.getNextNotification(latestID.getEventId()-1, 1, null); + AlterPartitionMessage alterPartitionMessage = deserializer.getAlterPartitionMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.ALTER_PARTITION, alterPartitionMessage.getEventType()); + assertThat(alterPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));// dbName + assertThat(alterPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));// tableName + //Location information, not sure how can we get this? Refresh all paths for every alter table add partition? + } +} + http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDbNotificationListenerSentryDeserializer.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDbNotificationListenerSentryDeserializer.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDbNotificationListenerSentryDeserializer.java new file mode 100644 index 0000000..8e588b1 --- /dev/null +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestDbNotificationListenerSentryDeserializer.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.tests.e2e.metastore; + +import org.junit.BeforeClass; + + +/** + * Make sure we are able to capture all HMS object and path changes using Hive's DbNotificationListener and + * Sentry's JSON deserializer. This would make sure Sentry is able to read the Notification logs written by + * Hive's DBNotificationListener + */ +public class TestDbNotificationListenerSentryDeserializer extends TestSentryListenerSentryDeserializer { + + @BeforeClass + public static void setupTestStaticConfiguration() throws Exception { + setMetastoreListener = true; + useDbNotificationListener = true; + AbstractMetastoreTestWithStaticConfiguration.setupTestStaticConfiguration(); + setupClass(); + } +} + http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestHMSNotificationLogUsingDBNotificationListener.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestHMSNotificationLogUsingDBNotificationListener.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestHMSNotificationLogUsingDBNotificationListener.java deleted file mode 100644 index 0b328d4..0000000 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestHMSNotificationLogUsingDBNotificationListener.java +++ /dev/null @@ -1,351 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.sentry.tests.e2e.metastore; - -import com.google.common.collect.Lists; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.api.*; -import org.apache.hive.hcatalog.messaging.CreateDatabaseMessage; -import org.apache.hive.hcatalog.messaging.HCatEventMessage; -import org.apache.hive.hcatalog.messaging.MessageDeserializer; -import org.apache.hive.hcatalog.messaging.MessageFactory; -import org.apache.hive.hcatalog.messaging.CreateTableMessage; -import org.apache.hive.hcatalog.messaging.DropTableMessage; -import org.apache.hive.hcatalog.messaging.AlterTableMessage; -import org.apache.hive.hcatalog.messaging.AlterPartitionMessage; -import org.apache.hive.hcatalog.messaging.DropDatabaseMessage; -import org.apache.hive.hcatalog.messaging.AddPartitionMessage; -import org.apache.hive.hcatalog.messaging.DropPartitionMessage; -import org.apache.sentry.provider.file.PolicyFile; -import org.apache.sentry.tests.e2e.hive.StaticUserGroup; -import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory; -import org.hamcrest.text.IsEqualIgnoringCase; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; - -import org.junit.*; - -import java.io.File; -import java.util.ArrayList; -import java.util.Random; - -/** - * Make sure NotificationLog is capturing the information correctly for the commands which change <Obj,Location> mapping - */ -public class TestHMSNotificationLogUsingDBNotificationListener extends AbstractMetastoreTestWithStaticConfiguration { - - private PolicyFile policyFile; - - private static HiveMetaStoreClient client; - private static MessageDeserializer deserializer; - private static Random random = new Random(); - - - @BeforeClass - public static void setupTestStaticConfiguration() throws Exception { - setMetastoreListener = true; - useDbNotificationListener = true; - AbstractMetastoreTestWithStaticConfiguration.setupTestStaticConfiguration(); - client = context.getMetaStoreClient(ADMIN1); - deserializer = MessageFactory.getDeserializer("json", ""); - } - - @AfterClass - public static void cleanupAfterClass() throws Exception { - if (client != null) { - client.close(); - } - } - - @Override - @Before - public void setup() throws Exception { - policyFile = setAdminOnServer1(ADMINGROUP); - policyFile.setUserGroupMapping(StaticUserGroup.getStaticMapping()); - writePolicyFile(policyFile); - super.setup(); - } - - @Test - public void testCreateDropDatabase() throws Exception { - CurrentNotificationEventId latestID, previousID; - NotificationEventResponse response; - - String testDB = "N_db" + random.nextInt(Integer.SIZE - 1); - - // Create database - // We need: - // - Dbname - // - location - createMetastoreDB(client, testDB); - latestID = client.getCurrentNotificationEventId(); - response = client.getNextNotification(latestID.getEventId() - 1, 1, null); - CreateDatabaseMessage createDatabaseMessage = deserializer.getCreateDatabaseMessage(response.getEvents().get(0).getMessage()); - assertEquals(HCatEventMessage.EventType.CREATE_DATABASE, createDatabaseMessage.getEventType()); //Validate EventType - assertEquals(testDB, createDatabaseMessage.getDB()); //dbName - //Location information is not available - - //Alter database location and rename are not supported. See HIVE-4847 - - //Drop database - // We need: - // - dbName - // - location - client.dropDatabase(testDB); - previousID = latestID; - latestID = client.getCurrentNotificationEventId(); - assertEquals(previousID.getEventId() + 1, latestID.getEventId()); //Validate monotonically increasing eventID - response = client.getNextNotification(latestID.getEventId() - 1, 1, null); - DropDatabaseMessage dropDatabaseMessage = deserializer.getDropDatabaseMessage(response.getEvents().get(0).getMessage()); - assertEquals(HCatEventMessage.EventType.DROP_DATABASE, dropDatabaseMessage.getEventType()); //Event type - assertThat(dropDatabaseMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB)); // dbName - //Location information is not available, but we might not really need it as we can drop all paths associated with - //the object when we drop - } - - @Test - public void testCreateDropTableWithPartition() throws Exception { - String testDB = "N_db" + random.nextInt(Integer.SIZE - 1); - String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); - - NotificationEventResponse response; - CurrentNotificationEventId latestID, previousID; - // Create database - createMetastoreDB(client, testDB); - - // Create table with partition - // We need: - // - dbname - // - tablename - // - location - createMetastoreTableWithPartition(client, testDB, - testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")), - Lists.newArrayList(new FieldSchema("part_col1", "string", ""))); - latestID = client.getCurrentNotificationEventId(); - response = client.getNextNotification(latestID.getEventId() - 1, 1, null); - CreateTableMessage createTableMessage = deserializer.getCreateTableMessage(response.getEvents().get(0).getMessage()); - assertEquals(HCatEventMessage.EventType.CREATE_TABLE, createTableMessage.getEventType()); - assertEquals(testDB, createTableMessage.getDB()); //dbName - assertEquals(testTable, createTableMessage.getTable()); //tableName - //Location information is not available - - //Drop table - // We need: - // - dbName - // - tableName - // - location - client.dropTable(testDB, testTable); - previousID = latestID; - latestID = client.getCurrentNotificationEventId(); - assertEquals(previousID.getEventId() + 1, latestID.getEventId()); - response = client.getNextNotification(latestID.getEventId() - 1, 1, null); - DropTableMessage dropTableMessage = deserializer.getDropTableMessage(response.getEvents().get(0).getMessage()); - assertEquals(HCatEventMessage.EventType.DROP_TABLE, dropTableMessage.getEventType()); - assertThat(dropTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName - assertThat(dropTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName - //Location information is not available, but we might not really need it as we can drop all paths associated with - //the object when we drop - } - - @Test - public void testCreateDropTableWithoutPartition() throws Exception { - String testDB = "N_db" + random.nextInt(Integer.SIZE - 1); - String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); - - NotificationEventResponse response; - CurrentNotificationEventId latestID, previousID; - // Create database - createMetastoreDB(client, testDB); - - // Create table with partition - // We need: - // - dbname - // - tablename - // - location - createMetastoreTable(client, testDB, testTable, Lists.newArrayList(new FieldSchema("col1", "int", ""))); - latestID = client.getCurrentNotificationEventId(); - response = client.getNextNotification(latestID.getEventId() - 1, 1, null); - CreateTableMessage createTableMessage = deserializer.getCreateTableMessage(response.getEvents().get(0).getMessage()); - assertEquals(HCatEventMessage.EventType.CREATE_TABLE, createTableMessage.getEventType()); - assertEquals(testDB, createTableMessage.getDB()); //dbName - assertEquals(testTable, createTableMessage.getTable()); //tableName - //Location information is not available - - //Drop table - // We need: - // - dbName - // - tableName - // - location - client.dropTable(testDB, testTable); - previousID = latestID; - latestID = client.getCurrentNotificationEventId(); - assertEquals(previousID.getEventId() + 1, latestID.getEventId()); - response = client.getNextNotification(latestID.getEventId() - 1, 1, null); - DropTableMessage dropTableMessage = deserializer.getDropTableMessage(response.getEvents().get(0).getMessage()); - assertEquals(HCatEventMessage.EventType.DROP_TABLE, dropTableMessage.getEventType()); - assertThat(dropTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName - assertThat(dropTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName - //Location information is not available, but we might not really need it as we can drop all paths associated with - //the object when we drop - } - - @Test - public void testAddDropPartition() throws Exception { - String testDB = "N_db" + random.nextInt(Integer.SIZE - 1); - String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); - - NotificationEventResponse response; - CurrentNotificationEventId latestID, previousID; - // Create database and table - createMetastoreDB(client, testDB); - Table tbl1 = createMetastoreTableWithPartition(client, testDB, testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")), - Lists.newArrayList(new FieldSchema("part_col1", "string", ""))); - - ArrayList<String> partVals1 = Lists.newArrayList("part1"); - - //Add partition - // We need: - // - dbName - // - tableName - // - partition location - addPartition(client, testDB, testTable, partVals1, tbl1); - latestID = client.getCurrentNotificationEventId(); - response = client.getNextNotification(latestID.getEventId() - 1, 1, null); - AddPartitionMessage addPartitionMessage = deserializer.getAddPartitionMessage(response.getEvents().get(0).getMessage()); - assertEquals(HCatEventMessage.EventType.ADD_PARTITION, addPartitionMessage.getEventType()); - assertThat(addPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));// dbName (returns lowered version) - assertThat(addPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));// tableName (returns lowered version) - //Location information is not available - - //Drop partition - // We need: - // - dbName - // - tableName - // - partition location - dropPartition(client, testDB, testTable, partVals1); - previousID = latestID; - latestID = client.getCurrentNotificationEventId(); - assertEquals(previousID.getEventId() + 1, latestID.getEventId()); - response = client.getNextNotification(latestID.getEventId() - 1, 1, null); - DropPartitionMessage dropPartitionMessage = deserializer.getDropPartitionMessage(response.getEvents().get(0).getMessage()); - assertEquals(HCatEventMessage.EventType.DROP_PARTITION, dropPartitionMessage.getEventType()); - assertThat(dropPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB)); //dbName - assertThat(dropPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable)); //tableName - //Location information is not available - - } - - @Ignore("Needs Hive >= 1.1.2") - @Test - public void testAlterTableWithPartition() throws Exception { - String testDB = "N_db" + random.nextInt(Integer.SIZE - 1); - String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); - - NotificationEventResponse response; - CurrentNotificationEventId latestID, previousID; - // Create database - createMetastoreDB(client, testDB); - - // Create table with partition - Table tbl1 = createMetastoreTableWithPartition(client, testDB, - testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")), - Lists.newArrayList(new FieldSchema("part_col1", "string", ""))); - - //Alter table location - // We need: - // - dbName - // - tableName - // - old location - // - new location - String tabDir1 = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR) - + File.separator + random.nextInt(Integer.SIZE - 1); - alterTableWithLocation(client, tbl1, tabDir1); - latestID = client.getCurrentNotificationEventId(); - response = client.getNextNotification(latestID.getEventId()-1, 1, null); - AlterTableMessage alterTableMessage = deserializer.getAlterTableMessage(response.getEvents().get(0).getMessage()); - assertEquals(HCatEventMessage.EventType.ALTER_TABLE, alterTableMessage.getEventType()); - assertThat(alterTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName - assertThat(alterTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName - //Old location is not available: This information is lost if not captured at the time of event. - //New location is not available - - //Alter table rename managed table - location also changes - // We need: - // - oldDbName - // - newDbName - // - oldTableName - // - newTableName - // - old location - // - new location - String newDBName = testDB + random.nextInt(Integer.SIZE - 1); - String newTableName = testTable + random.nextInt(Integer.SIZE - 1); - String newLocation = tabDir1 + random.nextInt(Integer.SIZE - 1); - createMetastoreDB(client, newDBName); - alterTableRename(client, tbl1, newDBName, newTableName, newLocation); - previousID = latestID; - latestID = client.getCurrentNotificationEventId(); - assertEquals(previousID.getEventId() + 1, latestID.getEventId()); - response = client.getNextNotification(latestID.getEventId()-1, 1, null); - alterTableMessage = deserializer.getAlterTableMessage(response.getEvents().get(0).getMessage()); - assertEquals(HCatEventMessage.EventType.ALTER_TABLE, alterTableMessage.getEventType()); - assertThat(alterTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//oldDbName - assertThat(alterTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//oldTableName - assertThat(response.getEvents().get(0).getDbName(), IsEqualIgnoringCase.equalToIgnoringCase(newDBName));//newDbName - assertThat(response.getEvents().get(0).getTableName(), IsEqualIgnoringCase.equalToIgnoringCase(newTableName));//newTableName - //Old location: This information is lost if not captured at the time of event. - //New location: Not sure how can we get this? Refresh all paths for every alter table add partition? - } - - @Ignore("Needs Hive >= 1.1.2") - @Test - public void testAlterPartition() throws Exception { - String testDB = "N_db" + random.nextInt(Integer.SIZE - 1); - String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); - - NotificationEventResponse response; - CurrentNotificationEventId latestID; - // Create database - createMetastoreDB(client, testDB); - - // Create table with partition - Table tbl1 = createMetastoreTableWithPartition(client, testDB, - testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")), - Lists.newArrayList(new FieldSchema("part_col1", "string", ""))); - ArrayList<String> partVals1 = Lists.newArrayList("part1"); - Partition partition = addPartition(client, testDB, testTable, partVals1, tbl1); - - - String warehouseDir = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR); - //Alter partition with location - // We need: - // - dbName - // - tableName - // - partition location - alterPartitionWithLocation(client, partition, warehouseDir + File.separator + "newpart"); - latestID = client.getCurrentNotificationEventId(); - response = client.getNextNotification(latestID.getEventId()-1, 1, null); - AlterPartitionMessage alterPartitionMessage = deserializer.getAlterPartitionMessage(response.getEvents().get(0).getMessage()); - assertEquals(HCatEventMessage.EventType.ALTER_PARTITION, alterPartitionMessage.getEventType()); - assertThat(alterPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));// dbName - assertThat(alterPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));// tableName - //Location information, not sure how can we get this? Refresh all paths for every alter table add partition? - } -} - http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java new file mode 100644 index 0000000..c4be62d --- /dev/null +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerInBuiltDeserializer.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.tests.e2e.metastore; + +import org.junit.BeforeClass; + +/** + * Make sure we are able to capture all HMS object and path changes using Sentry's SentryMetastorePostEventListener + * and Hive's inbuilt Notification log deserializer. This would make sure Sentry is not breaking other users of + * NotificationLog who might be using Hive's in built serializer + */ +public class TestSentryListenerInBuiltDeserializer extends TestDBNotificationListenerInBuiltDeserializer { + + @BeforeClass + public static void setupTestStaticConfiguration() throws Exception { + setMetastoreListener = true; + useDbNotificationListener = false; + beforeClass(); + } +} + http://git-wip-us.apache.org/repos/asf/sentry/blob/73d77bcd/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java new file mode 100644 index 0000000..6f1886f --- /dev/null +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java @@ -0,0 +1,375 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.tests.e2e.metastore; + +import com.google.common.collect.Lists; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.*; +import org.apache.hive.hcatalog.messaging.HCatEventMessage; +import org.apache.sentry.binding.metastore.messaging.json.*; +import org.apache.sentry.tests.e2e.hive.StaticUserGroup; +import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory; +import org.hamcrest.text.IsEqualIgnoringCase; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +import org.junit.*; + +import java.io.File; +import java.util.ArrayList; +import java.util.Random; + +/** + * Make sure we are able to capture all HMS object and path changes using Sentry's SentryMetastorePostEventListener + * and Sentry Notification log deserializer. Can be removed if we move to using DBNotificationListener + */ +public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestWithStaticConfiguration { + + protected static HiveMetaStoreClient client; + protected static SentryJSONMessageDeserializer deserializer; + protected static Random random = new Random(); + private static String warehouseDir; + private static String testDB; + + + @BeforeClass + public static void setupTestStaticConfiguration() throws Exception { + setMetastoreListener = true; + useDbNotificationListener = false; + AbstractMetastoreTestWithStaticConfiguration.setupTestStaticConfiguration(); + setupClass(); + } + + protected static void setupClass() throws Exception{ + client = context.getMetaStoreClient(ADMIN1); + deserializer = new SentryJSONMessageDeserializer(); + warehouseDir = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR); + writePolicyFile(setAdminOnServer1(ADMINGROUP).setUserGroupMapping(StaticUserGroup.getStaticMapping())); + + } + + @AfterClass + public static void cleanupAfterClass() throws Exception { + if (client != null) { + client.close(); + } + } + + @After + public void dropDBAfterTest() throws Exception { + if(client != null && testDB != null) { + dropMetastoreDBIfExists(client, testDB); + } + } + + @Test + public void testCreateDropDatabase() throws Exception { + CurrentNotificationEventId latestID, previousID; + NotificationEventResponse response; + + testDB = "N_db" + random.nextInt(Integer.SIZE - 1); + + // Create database + // We need: + // - Dbname + // - location + createMetastoreDB(client, testDB); + latestID = client.getCurrentNotificationEventId(); + response = client.getNextNotification(latestID.getEventId() - 1, 1, null); + SentryJSONCreateDatabaseMessage createDatabaseMessage = deserializer.getCreateDatabaseMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.CREATE_DATABASE, createDatabaseMessage.getEventType()); //Validate EventType + assertEquals(testDB, createDatabaseMessage.getDB()); //dbName + String expectedLocation = warehouseDir + "/" + testDB + ".db"; + if(!useDbNotificationListener) { + Assert.assertEquals(expectedLocation.toLowerCase(), createDatabaseMessage.getLocation()); + } + + //Alter database location and rename are not supported. See HIVE-4847 + + //Drop database + // We need: + // - dbName + // - location + client.dropDatabase(testDB); + previousID = latestID; + latestID = client.getCurrentNotificationEventId(); + assertEquals(previousID.getEventId() + 1, latestID.getEventId()); //Validate monotonically increasing eventID + response = client.getNextNotification(latestID.getEventId() - 1, 1, null); + SentryJSONDropDatabaseMessage dropDatabaseMessage = deserializer.getDropDatabaseMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.DROP_DATABASE, dropDatabaseMessage.getEventType()); //Event type + assertThat(dropDatabaseMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB)); // dbName + if(!useDbNotificationListener) { + Assert.assertEquals(expectedLocation.toLowerCase(), dropDatabaseMessage.getLocation()); //location + } + } + + @Test + public void testCreateDropTableWithPartition() throws Exception { + testDB = "N_db" + random.nextInt(Integer.SIZE - 1); + String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); + + NotificationEventResponse response; + CurrentNotificationEventId latestID, previousID; + // Create database + createMetastoreDB(client, testDB); + + // Create table with partition + // We need: + // - dbname + // - tablename + // - location + createMetastoreTableWithPartition(client, testDB, + testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")), + Lists.newArrayList(new FieldSchema("part_col1", "string", ""))); + latestID = client.getCurrentNotificationEventId(); + response = client.getNextNotification(latestID.getEventId() - 1, 1, null); + SentryJSONCreateTableMessage createTableMessage = deserializer.getCreateTableMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.CREATE_TABLE, createTableMessage.getEventType()); + assertEquals(testDB, createTableMessage.getDB()); //dbName + assertEquals(testTable, createTableMessage.getTable()); //tableName + String expectedLocation = warehouseDir + "/" + testDB + ".db/" + testTable; + if(!useDbNotificationListener) { + Assert.assertEquals(expectedLocation.toLowerCase(), createTableMessage.getLocation()); + } + + + //Drop table + // We need: + // - dbName + // - tableName + // - location + client.dropTable(testDB, testTable); + previousID = latestID; + latestID = client.getCurrentNotificationEventId(); + assertEquals(previousID.getEventId() + 1, latestID.getEventId()); + response = client.getNextNotification(latestID.getEventId() - 1, 1, null); + SentryJSONDropTableMessage dropTableMessage = deserializer.getDropTableMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.DROP_TABLE, dropTableMessage.getEventType()); + assertThat(dropTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName + assertThat(dropTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName + if(!useDbNotificationListener) { + Assert.assertEquals(expectedLocation.toLowerCase(), dropTableMessage.getLocation()); //location + } + } + + @Test + public void testCreateDropTableWithoutPartition() throws Exception { + testDB = "N_db" + random.nextInt(Integer.SIZE - 1); + String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); + + NotificationEventResponse response; + CurrentNotificationEventId latestID, previousID; + // Create database + createMetastoreDB(client, testDB); + + // Create table with partition + // We need: + // - dbname + // - tablename + // - location + createMetastoreTable(client, testDB, testTable, Lists.newArrayList(new FieldSchema("col1", "int", ""))); + latestID = client.getCurrentNotificationEventId(); + response = client.getNextNotification(latestID.getEventId() - 1, 1, null); + SentryJSONCreateTableMessage createTableMessage = deserializer.getCreateTableMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.CREATE_TABLE, createTableMessage.getEventType()); + assertEquals(testDB, createTableMessage.getDB()); //dbName + assertEquals(testTable, createTableMessage.getTable()); //tableName + String expectedLocation = warehouseDir + "/" + testDB + ".db/" + testTable; + if(!useDbNotificationListener) { + Assert.assertEquals(expectedLocation.toLowerCase(), createTableMessage.getLocation()); + } + + //Drop table + // We need: + // - dbName + // - tableName + // - location + client.dropTable(testDB, testTable); + previousID = latestID; + latestID = client.getCurrentNotificationEventId(); + assertEquals(previousID.getEventId() + 1, latestID.getEventId()); + response = client.getNextNotification(latestID.getEventId() - 1, 1, null); + SentryJSONDropTableMessage dropTableMessage = deserializer.getDropTableMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.DROP_TABLE, dropTableMessage.getEventType()); + assertThat(dropTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName + assertThat(dropTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName + if(!useDbNotificationListener) { + Assert.assertEquals(expectedLocation.toLowerCase(), dropTableMessage.getLocation()); //location + } + } + + @Test + public void testAddDropPartition() throws Exception { + testDB = "N_db" + random.nextInt(Integer.SIZE - 1); + String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); + + NotificationEventResponse response; + CurrentNotificationEventId latestID, previousID; + // Create database and table + createMetastoreDB(client, testDB); + Table tbl1 = createMetastoreTableWithPartition(client, testDB, testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")), + Lists.newArrayList(new FieldSchema("part_col1", "string", ""))); + + ArrayList<String> partVals1 = Lists.newArrayList("part1"); + + //Add partition + // We need: + // - dbName + // - tableName + // - partition location + addPartition(client, testDB, testTable, partVals1, tbl1); + latestID = client.getCurrentNotificationEventId(); + response = client.getNextNotification(latestID.getEventId() - 1, 1, null); + SentryJSONAddPartitionMessage addPartitionMessage = deserializer.getAddPartitionMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.ADD_PARTITION, addPartitionMessage.getEventType()); + assertThat(addPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));// dbName (returns lowered version) + assertThat(addPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));// tableName (returns lowered version) + String expectedLocation = warehouseDir + "/" + testDB + ".db/" + testTable ; //TODO: SENTRY-1387: Tablelocation is stored instead of partition location + if(!useDbNotificationListener) { + Assert.assertEquals(expectedLocation.toLowerCase(), addPartitionMessage.getLocations().get(0)); + } + + //Drop partition + // We need: + // - dbName + // - tableName + // - partition location + dropPartition(client, testDB, testTable, partVals1); + previousID = latestID; + latestID = client.getCurrentNotificationEventId(); + assertEquals(previousID.getEventId() + 1, latestID.getEventId()); + response = client.getNextNotification(latestID.getEventId() - 1, 1, null); + SentryJSONDropPartitionMessage dropPartitionMessage = deserializer.getDropPartitionMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.DROP_PARTITION, dropPartitionMessage.getEventType()); + assertThat(dropPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB)); //dbName + assertThat(dropPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable)); //tableName + if(!useDbNotificationListener) { + Assert.assertEquals(expectedLocation.toLowerCase(), dropPartitionMessage.getLocation()); + } + + } + + @Test + public void testAlterTableWithPartition() throws Exception { + testDB = "N_db" + random.nextInt(Integer.SIZE - 1); + String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); + + NotificationEventResponse response; + CurrentNotificationEventId latestID, previousID; + // Create database + createMetastoreDB(client, testDB); + + // Create table with partition + Table tbl1 = createMetastoreTableWithPartition(client, testDB, + testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")), + Lists.newArrayList(new FieldSchema("part_col1", "string", ""))); + String oldLocation = tbl1.getSd().getLocation(); + + //Alter table location + // We need: + // - dbName + // - tableName + // - old location + // - new location + String tabDir1 = hiveServer.getProperty(HiveServerFactory.WAREHOUSE_DIR) + + File.separator + random.nextInt(Integer.SIZE - 1); + alterTableWithLocation(client, tbl1, tabDir1); + latestID = client.getCurrentNotificationEventId(); + response = client.getNextNotification(latestID.getEventId()-1, 1, null); + SentryJSONAlterTableMessage alterTableMessage = deserializer.getAlterTableMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.ALTER_TABLE, alterTableMessage.getEventType()); + assertThat(alterTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//dbName + assertThat(alterTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//tableName + if(!useDbNotificationListener) { + Assert.assertEquals(oldLocation, alterTableMessage.getOldLocation()); //oldLocation + Assert.assertEquals(tbl1.getSd().getLocation(), alterTableMessage.getLocation()); //newLocation + } + + //Alter table rename managed table - location also changes + // We need: + // - oldDbName + // - newDbName + // - oldTableName + // - newTableName + // - old location + // - new location + oldLocation = tbl1.getSd().getLocation(); + String newDBName = testDB + random.nextInt(Integer.SIZE - 1); + String newTableName = testTable + random.nextInt(Integer.SIZE - 1); + String newLocation = tabDir1 + random.nextInt(Integer.SIZE - 1); + createMetastoreDB(client, newDBName); + previousID = latestID; + latestID = client.getCurrentNotificationEventId(); + assertEquals(previousID.getEventId() + 1, latestID.getEventId()); + alterTableRename(client, tbl1, newDBName, newTableName, newLocation); + previousID = latestID; + latestID = client.getCurrentNotificationEventId(); + assertEquals(previousID.getEventId() + 1, latestID.getEventId()); + response = client.getNextNotification(latestID.getEventId()-1, 1, null); + alterTableMessage = deserializer.getAlterTableMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.ALTER_TABLE, alterTableMessage.getEventType()); + assertThat(alterTableMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));//oldDbName + assertThat(alterTableMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));//oldTableName + assertThat(response.getEvents().get(0).getDbName(), IsEqualIgnoringCase.equalToIgnoringCase(newDBName));//newDbName + assertThat(response.getEvents().get(0).getTableName(), IsEqualIgnoringCase.equalToIgnoringCase(newTableName));//newTableName + if(!useDbNotificationListener) { + Assert.assertEquals(oldLocation, alterTableMessage.getOldLocation()); //oldLocation + Assert.assertEquals(tbl1.getSd().getLocation(), alterTableMessage.getLocation()); //newLocation + } + } + + @Test + public void testAlterPartition() throws Exception { + testDB = "N_db" + random.nextInt(Integer.SIZE - 1); + String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); + + NotificationEventResponse response; + CurrentNotificationEventId latestID; + // Create database + createMetastoreDB(client, testDB); + + // Create table with partition + Table tbl1 = createMetastoreTableWithPartition(client, testDB, + testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")), + Lists.newArrayList(new FieldSchema("part_col1", "string", ""))); + ArrayList<String> partVals1 = Lists.newArrayList("part1"); + Partition partition = addPartition(client, testDB, testTable, partVals1, tbl1); + + //Alter partition with location + // We need: + // - dbName + // - tableName + // - partition location + String oldLocation = tbl1.getSd().getLocation(); //TODO: SENTRY-1387: Tablelocation is stored instead of partition location + String newLocation = warehouseDir + File.separator + "newpart"; + alterPartitionWithLocation(client, partition, newLocation); + latestID = client.getCurrentNotificationEventId(); + response = client.getNextNotification(latestID.getEventId()-1, 1, null); + SentryJSONAlterPartitionMessage alterPartitionMessage = deserializer.getAlterPartitionMessage(response.getEvents().get(0).getMessage()); + assertEquals(HCatEventMessage.EventType.ALTER_PARTITION, alterPartitionMessage.getEventType()); + assertThat(alterPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));// dbName + assertThat(alterPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));// tableName + if(!useDbNotificationListener) { + Assert.assertEquals(oldLocation.toLowerCase(), alterPartitionMessage.getOldLocation()); + Assert.assertEquals(newLocation.toLowerCase(), alterPartitionMessage.getLocation()); + } + } +} +
