Repository: ranger Updated Branches: refs/heads/master fbc7f19db -> e4122919c
RANGER-1357: tag objects are not removed when attempting a full sync with Atlas Signed-off-by: Velmurugan Periasamy <v...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/e4122919 Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/e4122919 Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/e4122919 Branch: refs/heads/master Commit: e4122919c46bd9fec38a76b27c7e0f5fb98a5f5b Parents: fbc7f19 Author: Abhay Kulkarni <akulka...@hortonworks.com> Authored: Wed Feb 1 17:18:20 2017 -0800 Committer: Velmurugan Periasamy <v...@apache.org> Committed: Fri Feb 10 18:41:09 2017 -0500 ---------------------------------------------------------------------- .../source/atlas/AtlasNotificationMapper.java | 44 ++++++++++++-------- .../source/atlasrest/AtlasRESTTagSource.java | 8 ++-- .../tagsync/source/atlasrest/AtlasRESTUtil.java | 28 ++++++------- 3 files changed, 44 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ranger/blob/e4122919/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java index ec8508e..922317e 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java @@ -108,7 +108,7 @@ public class AtlasNotificationMapper { return ret; } - public static Map<String, ServiceTags> processEntitiesWithTraits(List<AtlasEntityWithTraits> atlasEntities) { + public static Map<String, ServiceTags> processAtlasEntities(List<AtlasEntityWithTraits> atlasEntities) { Map<String, ServiceTags> ret = null; try { @@ -245,32 +245,40 @@ public class AtlasNotificationMapper { RangerServiceResource serviceResource = AtlasResourceMapperUtil.getRangerServiceResource(entity); if (serviceResource != null) { - List<RangerTag> tags = getTags(entityWithTraits); - List<RangerTagDef> tagDefs = getTagDefs(entityWithTraits); - String serviceName = serviceResource.getServiceName(); + List<RangerTag> tags = getTags(entityWithTraits); + List<RangerTagDef> tagDefs = getTagDefs(entityWithTraits); + String serviceName = serviceResource.getServiceName(); ret = createOrGetServiceTags(serviceTagsMap, serviceName); - serviceResource.setId((long)ret.getServiceResources().size()); - ret.getServiceResources().add(serviceResource); + if (serviceTagsMap == null || CollectionUtils.isNotEmpty(tags)) { + + serviceResource.setId((long) ret.getServiceResources().size()); + ret.getServiceResources().add(serviceResource); - List<Long> tagIds = new ArrayList<Long>(); + List<Long> tagIds = new ArrayList<Long>(); - if(CollectionUtils.isNotEmpty(tags)) { - for(RangerTag tag : tags) { - tag.setId((long)ret.getTags().size()); - ret.getTags().put(tag.getId(), tag); + if (CollectionUtils.isNotEmpty(tags)) { + for (RangerTag tag : tags) { + tag.setId((long) ret.getTags().size()); + ret.getTags().put(tag.getId(), tag); - tagIds.add(tag.getId()); + tagIds.add(tag.getId()); + } } - } - ret.getResourceToTagIds().put(serviceResource.getId(), tagIds); + ret.getResourceToTagIds().put(serviceResource.getId(), tagIds); - if(CollectionUtils.isNotEmpty(tagDefs)) { - for(RangerTagDef tagDef : tagDefs) { - tagDef.setId((long)ret.getTagDefinitions().size()); - ret.getTagDefinitions().put(tagDef.getId(), tagDef); + if (CollectionUtils.isNotEmpty(tagDefs)) { + for (RangerTagDef tagDef : tagDefs) { + tagDef.setId((long) ret.getTagDefinitions().size()); + ret.getTagDefinitions().put(tagDef.getId(), tagDef); + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Entity " + entityWithTraits + " does not have any tags associated with it when full-sync is being done."); + LOG.debug("Will not add this entity to serviceTags, so that this entity, if exists, will be removed from ranger"); } } } else { http://git-wip-us.apache.org/repos/asf/ranger/blob/e4122919/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java index e0976cd..7da4ed4 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java @@ -194,16 +194,16 @@ public class AtlasRESTTagSource extends AbstractTagSource implements Runnable { public void synchUp() { - List<AtlasEntityWithTraits> atlasEntitiesWithTraits = atlasRESTUtil.getEntitiesWithTraits(); + List<AtlasEntityWithTraits> atlasEntities = atlasRESTUtil.getAtlasEntities(); - if (CollectionUtils.isNotEmpty(atlasEntitiesWithTraits)) { + if (CollectionUtils.isNotEmpty(atlasEntities)) { if (LOG.isDebugEnabled()) { - for (AtlasEntityWithTraits element : atlasEntitiesWithTraits) { + for (AtlasEntityWithTraits element : atlasEntities) { LOG.debug(element); } } - Map<String, ServiceTags> serviceTagsMap = AtlasNotificationMapper.processEntitiesWithTraits(atlasEntitiesWithTraits); + Map<String, ServiceTags> serviceTagsMap = AtlasNotificationMapper.processAtlasEntities(atlasEntities); if (MapUtils.isNotEmpty(serviceTagsMap)) { for (Map.Entry<String, ServiceTags> entry : serviceTagsMap.entrySet()) { http://git-wip-us.apache.org/repos/asf/ranger/blob/e4122919/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java index bf09cfd..167fe68 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java @@ -88,9 +88,9 @@ public class AtlasRESTUtil { } } - public List<AtlasEntityWithTraits> getEntitiesWithTraits() { + public List<AtlasEntityWithTraits> getAtlasEntities() { if (LOG.isDebugEnabled()) { - LOG.debug("==> getEntriesWithTraits()"); + LOG.debug("==> getAtlasEntities()"); } List<AtlasEntityWithTraits> ret = new ArrayList<AtlasEntityWithTraits>(); @@ -129,9 +129,9 @@ public class AtlasRESTUtil { Map<String, Object> traitsAttribute = getAttribute(definition, TRAITS_ATTRIBUTE, Map.class); - if (MapUtils.isNotEmpty(traitsAttribute)) { + List<IStruct> allTraits = new LinkedList<>(); - List<IStruct> allTraits = new LinkedList<>(); + if (MapUtils.isNotEmpty(traitsAttribute)) { for (Map.Entry<String, Object> entry : traitsAttribute.entrySet()) { @@ -151,24 +151,24 @@ public class AtlasRESTUtil { allTraits.add(trait1); allTraits.addAll(superTypes); } + } - IReferenceableInstance entity = InstanceSerialization.fromJsonReferenceable(gson.toJson(definition), true); + IReferenceableInstance entity = InstanceSerialization.fromJsonReferenceable(gson.toJson(definition), true); - if (entity != null) { - AtlasEntityWithTraits entityWithTraits = new AtlasEntityWithTraits(entity, allTraits); - ret.add(entityWithTraits); - } else { - if (LOG.isInfoEnabled()) { - LOG.info("Could not create Atlas entity from its definition, type=" + type + ", guid=" + guid); - } + if (entity != null) { + AtlasEntityWithTraits atlasEntity = new AtlasEntityWithTraits(entity, allTraits); + ret.add(atlasEntity); + } else { + if (LOG.isInfoEnabled()) { + LOG.info("Could not create Atlas entity from its definition, type=" + type + ", guid=" + guid); } - } } + } if (LOG.isDebugEnabled()) { - LOG.debug("<== getEntriesWithTraits()"); + LOG.debug("<== getAtlasEntities()"); } }
