Repository: atlas Updated Branches: refs/heads/master cabc1e550 -> 6c49542b8
ATLAS-2581: V2 Hive hook notifications - incorrect location for sd after moving table to a different database Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/6c49542b Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/6c49542b Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/6c49542b Branch: refs/heads/master Commit: 6c49542b8e95b248c4a4062f13e499e2ebea93bf Parents: cabc1e5 Author: Madhan Neethiraj <mad...@apache.org> Authored: Tue Apr 17 15:10:42 2018 -0700 Committer: Madhan Neethiraj <mad...@apache.org> Committed: Wed Apr 18 09:43:44 2018 -0700 ---------------------------------------------------------------------- .../hive/hook/events/AlterTableRename.java | 53 +++++++++++++------- .../atlas/util/AtlasGremlin3QueryProvider.java | 2 +- 2 files changed, 35 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/6c49542b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java index eac095c..35b0586 100644 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTableRename.java @@ -72,6 +72,8 @@ public class AlterTableRename extends BaseHiveEvent { continue; } + newTable = getHive().getTable(newTable.getDbName(), newTable.getTableName()); + break; } } @@ -88,27 +90,27 @@ public class AlterTableRename extends BaseHiveEvent { // first update with oldTable info, so that the table will be created if it is not present in Atlas ret.add(new EntityUpdateRequestV2(getUserName(), new AtlasEntitiesWithExtInfo(oldTableEntity))); + AtlasEntityWithExtInfo renamedTableEntity = toTableEntity(newTable); + // update qualifiedName for all columns, partitionKeys, storageDesc - String newTableQualifiedName = getQualifiedName(newTable); + String renamedTableQualifiedName = (String) renamedTableEntity.getEntity().getAttribute(ATTRIBUTE_QUALIFIED_NAME); - renameColumns((List<AtlasObjectId>) oldTableEntity.getEntity().getAttribute(ATTRIBUTE_COLUMNS), oldTableEntity, newTableQualifiedName, ret); - renameColumns((List<AtlasObjectId>) oldTableEntity.getEntity().getAttribute(ATTRIBUTE_PARTITION_KEYS), oldTableEntity, newTableQualifiedName, ret); - renameStorageDesc((AtlasObjectId) oldTableEntity.getEntity().getAttribute(ATTRIBUTE_STORAGEDESC), oldTableEntity, newTableQualifiedName, ret); + renameColumns((List<AtlasObjectId>) oldTableEntity.getEntity().getAttribute(ATTRIBUTE_COLUMNS), oldTableEntity, renamedTableQualifiedName, ret); + renameColumns((List<AtlasObjectId>) oldTableEntity.getEntity().getAttribute(ATTRIBUTE_PARTITION_KEYS), oldTableEntity, renamedTableQualifiedName, ret); + renameStorageDesc(oldTableEntity, renamedTableEntity, ret); - // update qualifiedName and other attributes (like params - which include lastModifiedTime, lastModifiedBy) of the table - AtlasEntityWithExtInfo newTableEntity = toTableEntity(newTable); + // remove columns, partitionKeys and storageDesc - as they have already been updated above + removeAttribute(renamedTableEntity, ATTRIBUTE_COLUMNS); + removeAttribute(renamedTableEntity, ATTRIBUTE_PARTITION_KEYS); + removeAttribute(renamedTableEntity, ATTRIBUTE_STORAGEDESC); // set previous name as the alias - newTableEntity.getEntity().setAttribute(ATTRIBUTE_ALIASES, Collections.singletonList(oldTable.getTableName())); - - // remove columns, partitionKeys and storageDesc - as they have already been updated above - removeAttribute(newTableEntity, ATTRIBUTE_COLUMNS); - removeAttribute(newTableEntity, ATTRIBUTE_PARTITION_KEYS); - removeAttribute(newTableEntity, ATTRIBUTE_STORAGEDESC); + renamedTableEntity.getEntity().setAttribute(ATTRIBUTE_ALIASES, Collections.singletonList(oldTable.getTableName())); AtlasObjectId oldTableId = new AtlasObjectId(oldTableEntity.getEntity().getTypeName(), ATTRIBUTE_QUALIFIED_NAME, oldTableEntity.getEntity().getAttribute(ATTRIBUTE_QUALIFIED_NAME)); - ret.add(new EntityPartialUpdateRequestV2(getUserName(), oldTableId, newTableEntity)); + // update qualifiedName and other attributes (like params - which include lastModifiedTime, lastModifiedBy) of the table + ret.add(new EntityPartialUpdateRequestV2(getUserName(), oldTableId, renamedTableEntity)); context.removeFromKnownTable((String) oldTableEntity.getEntity().getAttribute(ATTRIBUTE_QUALIFIED_NAME)); @@ -127,11 +129,14 @@ public class AlterTableRename extends BaseHiveEvent { } } - private void renameStorageDesc(AtlasObjectId sdId, AtlasEntityExtInfo oldEntityExtInfo, String newTableQualifiedName, List<HookNotification> notifications) { - if (sdId != null) { - AtlasEntity oldSd = oldEntityExtInfo.getEntity(sdId.getGuid()); + private void renameStorageDesc(AtlasEntityWithExtInfo oldEntityExtInfo, AtlasEntityWithExtInfo newEntityExtInfo, List<HookNotification> notifications) { + AtlasEntity oldSd = getStorageDescEntity(oldEntityExtInfo); + AtlasEntity newSd = getStorageDescEntity(newEntityExtInfo); + + if (oldSd != null && newSd != null) { AtlasObjectId oldSdId = new AtlasObjectId(oldSd.getTypeName(), ATTRIBUTE_QUALIFIED_NAME, oldSd.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); - AtlasEntity newSd = new AtlasEntity(oldSd.getTypeName(), ATTRIBUTE_QUALIFIED_NAME, getStorageDescQualifiedName(newTableQualifiedName)); + + newSd.removeAttribute(ATTRIBUTE_TABLE); notifications.add(new EntityPartialUpdateRequestV2(getUserName(), oldSdId, new AtlasEntityWithExtInfo(newSd))); } @@ -156,7 +161,17 @@ public class AlterTableRename extends BaseHiveEvent { } } - private String getStorageDescQualifiedName(String tblQualifiedName) { - return tblQualifiedName + "_storage"; + private AtlasEntity getStorageDescEntity(AtlasEntityWithExtInfo tableEntity) { + AtlasEntity ret = null; + + if (tableEntity != null && tableEntity.getEntity() != null) { + Object attrSdId = tableEntity.getEntity().getAttribute(ATTRIBUTE_STORAGEDESC); + + if (attrSdId instanceof AtlasObjectId) { + ret = tableEntity.getReferredEntity(((AtlasObjectId) attrSdId).getGuid()); + } + } + + return ret; } } http://git-wip-us.apache.org/repos/asf/atlas/blob/6c49542b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java index e82c658..8555f70 100644 --- a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java +++ b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin3QueryProvider.java @@ -52,7 +52,7 @@ public class AtlasGremlin3QueryProvider extends AtlasGremlin2QueryProvider { case TO_RANGE_LIST: return ".range(startIdx, endIdx).toList()"; case RELATIONSHIP_SEARCH: - return "g.V().has('__guid', guid).both(relation).has('__state', within(states))"; + return "g.V().has('__guid', guid).bothE(relation).has('__state', within(states)).otherV().has('__state', within(states))"; case RELATIONSHIP_SEARCH_ASCENDING_SORT: return ".order().by(sortAttributeName, incr)"; case RELATIONSHIP_SEARCH_DESCENDING_SORT:
