Repository: incubator-atlas Updated Branches: refs/heads/master b8f4ffb68 -> 755e59c08
ATLAS-528 Support drop table,view (sumasai) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/755e59c0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/755e59c0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/755e59c0 Branch: refs/heads/master Commit: 755e59c08dbbc3b03fc2f6fc4908396433b97e88 Parents: b8f4ffb Author: Suma Shivaprasad <[email protected]> Authored: Sat Apr 9 09:53:58 2016 -0700 Committer: Suma Shivaprasad <[email protected]> Committed: Sat Apr 9 09:53:58 2016 -0700 ---------------------------------------------------------------------- .../atlas/hive/bridge/HiveMetaStoreBridge.java | 12 ++- .../org/apache/atlas/hive/hook/HiveHook.java | 22 +++++ .../org/apache/atlas/hive/hook/HiveHookIT.java | 88 +++++++++++++++++++- release-log.txt | 1 + 4 files changed, 115 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/755e59c0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java index eb5f1e6..6b348e2 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java @@ -335,8 +335,8 @@ public class HiveMetaStoreBridge { tableReference.set(HiveDataModelGenerator.COLUMNS, getColumns(hiveTable.getCols(), tableQualifiedName)); - // add reference to the StorageDescriptorx - Referenceable sdReferenceable = fillStorageDescStruct(hiveTable.getSd(), tableQualifiedName, tableQualifiedName); + // add reference to the StorageDescriptor + Referenceable sdReferenceable = fillStorageDesc(hiveTable.getSd(), tableQualifiedName, getStorageDescQFName(tableQualifiedName)); tableReference.set("sd", sdReferenceable); // add reference to the Partition Keys @@ -359,6 +359,10 @@ public class HiveMetaStoreBridge { return tableReference; } + private String getStorageDescQFName(String entityQualifiedName) { + return entityQualifiedName + "_storage"; + } + private Referenceable registerTable(Referenceable dbReference, Table table) throws Exception { String dbName = table.getDbName(); String tableName = table.getTableName(); @@ -410,7 +414,7 @@ public class HiveMetaStoreBridge { return new Referenceable(sd.getId().id, sd.getTypeName(), null); } - public Referenceable fillStorageDescStruct(StorageDescriptor storageDesc, String tableQualifiedName, + public Referenceable fillStorageDesc(StorageDescriptor storageDesc, String tableQualifiedName, String sdQualifiedName) throws Exception { LOG.debug("Filling storage descriptor information for " + storageDesc); @@ -466,7 +470,7 @@ public class HiveMetaStoreBridge { ref.set("path", pathUri); // Path path = new Path(pathUri); // ref.set("name", path.getName()); -// TODO - Fix after ATLAS-542 to shorter Name + //TODO - Fix after ATLAS-542 to shorter Name ref.set("name", pathUri); ref.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, pathUri); return ref; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/755e59c0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java index 4102263..505e5e7 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java @@ -308,6 +308,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { case EXPORT: case IMPORT: case QUERY: + case TRUNCATETABLE: registerProcess(dgiBridge, event); break; @@ -326,6 +327,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { case ALTERTABLE_ADDCOLS: case ALTERTABLE_REPLACECOLS: case ALTERTABLE_RENAMECOL: + case ALTERTABLE_PARTCOLTYPE: handleEventOutputs(dgiBridge, event, Type.TABLE); break; case ALTERTABLE_LOCATION: @@ -334,17 +336,37 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { //Track altered lineage in case of external tables handleExternalTables(dgiBridge, event, tablesUpdated.get(0).getLeft(), tablesUpdated.get(0).getRight()); } + break; case ALTERDATABASE: case ALTERDATABASE_OWNER: handleEventOutputs(dgiBridge, event, Type.DATABASE); break; + case DROPTABLE: + case DROPVIEW: + deleteTable(dgiBridge, event); + break; + default: } notifyEntities(messages); } + private void deleteTable(HiveMetaStoreBridge dgiBridge, HiveEventContext event) { + for (Entity output : event.outputs) { + if (Type.TABLE.equals(output.getType())) { + final String tblQualifiedName = HiveMetaStoreBridge.getTableQualifiedName(dgiBridge.getClusterName(), output.getTable().getDbName(), output.getTable().getTableName()); + LOG.info("Deleting table {} ", tblQualifiedName); + messages.add( + new HookNotification.EntityDeleteRequest(event.getUser(), + HiveDataTypes.HIVE_TABLE.getName(), + HiveDataModelGenerator.NAME, + tblQualifiedName)); + } + } + } + private void renameTable(HiveMetaStoreBridge dgiBridge, HiveEventContext event) throws Exception { //crappy, no easy of getting new name assert event.getInputs() != null && event.getInputs().size() == 1; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/755e59c0/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java index 8ef8479..7a09b47 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java @@ -497,7 +497,6 @@ public class HiveHookIT { runCommand(query); Referenceable processReference = validateProcess(query, 1, 1); validateHDFSPaths(processReference, filename, OUTPUTS); - validateInputTables(processReference, tableId); //Import @@ -510,7 +509,6 @@ public class HiveHookIT { validateHDFSPaths(processReference, filename, INPUTS); validateOutputTables(processReference, tableId); - } @Test @@ -541,8 +539,6 @@ public class HiveHookIT { validateHDFSPaths(processReference, filename, INPUTS); validateOutputTables(processReference, tableId); - - } @Test @@ -684,6 +680,41 @@ public class HiveHookIT { Assert.assertEquals(columns.get(0).get(HiveDataModelGenerator.NAME), "id"); } + @Test() + public void testTruncateTable() throws Exception { + String tableName = createTable(false); + String query = String.format("truncate table %s", tableName); + runCommand(query); + + String tableId = assertTableIsRegistered(DEFAULT_DB, tableName); + validateProcess(query, 0, 1); + + //Check lineage + String datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName); + JSONObject response = dgiCLient.getInputGraph(datasetName); + JSONObject vertices = response.getJSONObject("values").getJSONObject("vertices"); + //Below should be assertTrue - Fix https://issues.apache.org/jira/browse/ATLAS-653 + Assert.assertFalse(vertices.has(tableId)); + } + + @Test + public void testAlterTablePartitionColumnType() throws Exception { + String tableName = createTable(true, true, false); + final String newType = "int"; + String query = String.format("ALTER TABLE %s PARTITION COLUMN (dt %s)", tableName, newType); + runCommand(query); + + final String tableId = assertTableIsRegistered(DEFAULT_DB, tableName); + final String dtColId = assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), "dt")); + Referenceable table = dgiCLient.getEntity(tableId); + Referenceable column = dgiCLient.getEntity(dtColId); + Assert.assertEquals(column.get("type"), newType); + + final List<Referenceable> partitionKeys = (List<Referenceable>) table.get("partitionKeys"); + Assert.assertEquals(partitionKeys.size(), 1); + Assert.assertEquals(partitionKeys.get(0).getId()._getId(), dtColId); + } + @Test public void testAlterViewRename() throws Exception { String tableName = createTable(); @@ -834,7 +865,56 @@ public class HiveHookIT { //Add another property runSerdePropsQuery(tableName, expectedProps); + } + + @Test + public void testDropTable() throws Exception { + //Test Deletion of tables and its corrresponding columns + String tableName = createTable(true, true, false); + assertTableIsRegistered(DEFAULT_DB, tableName); + assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), "id")); + assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), "name")); + + final String query = String.format("drop table %s ", tableName); + + runCommand(query); + assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), "id")); + assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), "name")); + assertTableIsNotRegistered(DEFAULT_DB, tableName); + } + + @Test + public void testDropNonExistingTable() throws Exception { + //Test Deletion of a non existing table + final String tableName = "nonexistingtable"; + assertTableIsNotRegistered(DEFAULT_DB, tableName); + final String query = String.format("drop table if exists %s", tableName); + runCommand(query); + + //Should have no effect + assertTableIsNotRegistered(DEFAULT_DB, tableName); + assertProcessIsNotRegistered(query); + } + + @Test + public void testDropView() throws Exception { + //Test Deletion of tables and its corrresponding columns + String tableName = createTable(true, true, false); + String viewName = tableName(); + String query = "create view " + viewName + " as select * from " + tableName; + runCommand(query); + + assertTableIsRegistered(DEFAULT_DB, viewName); + assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName), "id")); + assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName), "name")); + + query = String.format("drop view %s ", viewName); + + runCommand(query); + assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName), "id")); + assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName), "name")); + assertTableIsNotRegistered(DEFAULT_DB, viewName); } private void runSerdePropsQuery(String tableName, Map<String, String> expectedProps) throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/755e59c0/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index a2fd4ea..2125f87 100644 --- a/release-log.txt +++ b/release-log.txt @@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-528 Support drop table,view (sumasai) ATLAS-603 Document High Availability of Atlas (yhemanth via sumasai) ATLAS-498 Support Embedded HBase (tbeerbower via sumasai) ATLAS-527 Support lineage for load table, import, export (sumasai via shwethags)
