Repository: sentry Updated Branches: refs/heads/sentry-ha-redesign 183af2c9b -> e840bdb15
SENTRY-1387: Add HDFS sync tests for drop partition for external/implicit locations (Nachiket Vaidya, Review by kalyan kumar kalvagadda, Alex Kolbasov and Hao Hao) Root cause: makeMetastoreBasePartitionObject() assigns the table location in partition. There are two versions of makeMetastorePartitionObject() that call makeMetastoreBasePartitionObject(). One sets given location and other keeps the location unchanged (which is table location). To fix this: set the partition location to null in makeMetastoreBasePartitionObject(). It will auto-generated by Hive in the format : <part_column>=<part_value>. Testing done: * ran complete test cases of sentry * modified unit test case. Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/e840bdb1 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/e840bdb1 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/e840bdb1 Branch: refs/heads/sentry-ha-redesign Commit: e840bdb1572d597205d9bbd85b80ff0f3a9f7c8c Parents: 183af2c Author: Alexander Kolbasov <[email protected]> Authored: Wed Feb 8 12:08:23 2017 -0800 Committer: Alexander Kolbasov <[email protected]> Committed: Wed Feb 8 12:08:23 2017 -0800 ---------------------------------------------------------------------- ...bstractMetastoreTestWithStaticConfiguration.java | 1 + .../TestSentryListenerSentryDeserializer.java | 16 ++++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/e840bdb1/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java index 567b4c8..18720eb 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/AbstractMetastoreTestWithStaticConfiguration.java @@ -188,6 +188,7 @@ public abstract class AbstractMetastoreTestWithStaticConfiguration extends part4.setValues(ptnVals); part4.setParameters(new HashMap<String, String>()); part4.setSd(tbl.getSd().deepCopy()); + part4.getSd().setLocation(null); part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy()); part4.setParameters(new HashMap<String, String>()); return part4; http://git-wip-us.apache.org/repos/asf/sentry/blob/e840bdb1/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java index 4d8889a..5a51d93 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/metastore/TestSentryListenerSentryDeserializer.java @@ -221,15 +221,17 @@ public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestW public void testAddDropPartition() throws Exception { testDB = "N_db" + random.nextInt(Integer.SIZE - 1); String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); + String partColName = "part_col1"; + String partColValue = "part1"; NotificationEventResponse response; CurrentNotificationEventId latestID, previousID; // Create database and table createMetastoreDB(client, testDB); Table tbl1 = createMetastoreTableWithPartition(client, testDB, testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")), - Lists.newArrayList(new FieldSchema("part_col1", "string", ""))); + Lists.newArrayList(new FieldSchema(partColName, "string", ""))); - ArrayList<String> partVals1 = Lists.newArrayList("part1"); + ArrayList<String> partVals1 = Lists.newArrayList(partColValue); //Add partition // We need: @@ -243,7 +245,7 @@ public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestW assertEquals(HCatEventMessage.EventType.ADD_PARTITION, addPartitionMessage.getEventType()); assertThat(addPartitionMessage.getDB(), IsEqualIgnoringCase.equalToIgnoringCase(testDB));// dbName (returns lowered version) assertThat(addPartitionMessage.getTable(), IsEqualIgnoringCase.equalToIgnoringCase(testTable));// tableName (returns lowered version) - String expectedLocation = warehouseDir + "/" + testDB + ".db/" + testTable ; //TODO: SENTRY-1387: Tablelocation is stored instead of partition location + String expectedLocation = warehouseDir + "/" + testDB + ".db/" + testTable + "/" + partColName + "=" + partColValue; if(!useDbNotificationListener) { Assert.assertEquals(expectedLocation.toLowerCase(), addPartitionMessage.getLocations().get(0)); } @@ -340,6 +342,8 @@ public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestW public void testAlterPartition() throws Exception { testDB = "N_db" + random.nextInt(Integer.SIZE - 1); String testTable = "N_table" + random.nextInt(Integer.SIZE - 1); + String partColName = "part_col1"; + String partColValue = "part1"; NotificationEventResponse response; CurrentNotificationEventId latestID; @@ -349,8 +353,8 @@ public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestW // Create table with partition Table tbl1 = createMetastoreTableWithPartition(client, testDB, testTable, Lists.newArrayList(new FieldSchema("col1", "int", "")), - Lists.newArrayList(new FieldSchema("part_col1", "string", ""))); - ArrayList<String> partVals1 = Lists.newArrayList("part1"); + Lists.newArrayList(new FieldSchema(partColName, "string", ""))); + ArrayList<String> partVals1 = Lists.newArrayList(partColValue); Partition partition = addPartition(client, testDB, testTable, partVals1, tbl1); //Alter partition with location @@ -358,7 +362,7 @@ public class TestSentryListenerSentryDeserializer extends AbstractMetastoreTestW // - dbName // - tableName // - partition location - String oldLocation = tbl1.getSd().getLocation(); //TODO: SENTRY-1387: Tablelocation is stored instead of partition location + String oldLocation = tbl1.getSd().getLocation() + "/" + partColName + "=" + partColValue; String newLocation = warehouseDir + File.separator + "newpart"; alterPartitionWithLocation(client, partition, newLocation); latestID = client.getCurrentNotificationEventId();
