Repository: incubator-ranger Updated Branches: refs/heads/master 6b22ecbac -> cba444bfd
Added handling for ENTITY_DELETED notification, rolled back 'None' tag-source, addressed review comments Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/cba444bf Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/cba444bf Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/cba444bf Branch: refs/heads/master Commit: cba444bfd621725339b14ee805967d73be849d14 Parents: 6b22ecb Author: Abhay Kulkarni <[email protected]> Authored: Wed Apr 13 15:54:39 2016 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Wed Apr 27 18:31:52 2016 -0700 ---------------------------------------------------------------------- pom.xml | 2 +- .../ranger/rest/ServiceTagsProcessor.java | 42 ++++++++++++----- .../conf/templates/installprop2xml.properties | 2 - .../conf/templates/ranger-tagsync-template.xml | 4 -- tagsync/scripts/setup.py | 2 +- .../apache/ranger/tagsync/model/TagSource.java | 2 - .../ranger/tagsync/process/TagSynchronizer.java | 2 +- .../source/atlas/AtlasNotificationMapper.java | 49 +++++++++++++++++--- .../tagsync/source/atlas/AtlasTagSource.java | 2 +- .../src/main/resources/ranger-tagsync-site.xml | 4 -- 10 files changed, 76 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index aef42c6..6377ed3 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,7 @@ <asm.all.version>3.2</asm.all.version> <asm.version>3.1</asm.version> <aspectj.version>1.8.2</aspectj.version> - <atlas.version>0.6-incubating-SNAPSHOT</atlas.version> + <atlas.version>0.7-incubating-SNAPSHOT</atlas.version> <avatica.version>1.7.1</avatica.version> <c3p0.version>0.9.1.2</c3p0.version> <calcite.version>0.9.2-incubating</calcite.version> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java index baafb0e..9c19bb0 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java @@ -428,8 +428,24 @@ public class ServiceTagsProcessor { if (CollectionUtils.isNotEmpty(serviceResources)) { for (RangerServiceResource serviceResource : serviceResources) { + + RangerServiceResource objToDelete = null; + try { - RangerServiceResource objToDelete = tagStore.getServiceResourceByGuid(serviceResource.getGuid()); + + if (StringUtils.isNotBlank(serviceResource.getGuid())) { + objToDelete = tagStore.getServiceResourceByGuid(serviceResource.getGuid()); + } + + if (objToDelete == null) { + if (MapUtils.isNotEmpty(serviceResource.getResourceElements())) { + RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(serviceResource); + + String serviceResourceSignature = serializer.getSignature(); + + objToDelete = tagStore.getServiceResourceByServiceAndResourceSignature(serviceResource.getServiceName(), serviceResourceSignature); + } + } if (objToDelete != null) { @@ -512,25 +528,27 @@ public class ServiceTagsProcessor { List<String> serviceResourcesInDb = tagStore.getServiceResourceGuidsByService(serviceTags.getServiceName()); - for (String dbServiceResourceGuid : serviceResourcesInDb) { + if (CollectionUtils.isNotEmpty(serviceResourcesInDb)) { + for (String dbServiceResourceGuid : serviceResourcesInDb) { - if (! serviceResourcesInServiceTagsMap.containsKey(dbServiceResourceGuid)) { + if (!serviceResourcesInServiceTagsMap.containsKey(dbServiceResourceGuid)) { - if (LOG.isDebugEnabled()) { - LOG.debug("Deleting serviceResource(guid=" + dbServiceResourceGuid + ") and its tag-associations..."); - } + if (LOG.isDebugEnabled()) { + LOG.debug("Deleting serviceResource(guid=" + dbServiceResourceGuid + ") and its tag-associations..."); + } - List<RangerTagResourceMap> tagResourceMaps = tagStore.getTagResourceMapsForResourceGuid(dbServiceResourceGuid); + List<RangerTagResourceMap> tagResourceMaps = tagStore.getTagResourceMapsForResourceGuid(dbServiceResourceGuid); - if (CollectionUtils.isNotEmpty(tagResourceMaps)) { - for (RangerTagResourceMap tagResourceMap : tagResourceMaps) { - tagStore.deleteTagResourceMap(tagResourceMap.getId()); + if (CollectionUtils.isNotEmpty(tagResourceMaps)) { + for (RangerTagResourceMap tagResourceMap : tagResourceMaps) { + tagStore.deleteTagResourceMap(tagResourceMap.getId()); + } } + + tagStore.deleteServiceResourceByGuid(dbServiceResourceGuid); } - tagStore.deleteServiceResourceByGuid(dbServiceResourceGuid); } - } // Add/update resources and other tag-model objects provided in service-tags http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/tagsync/conf/templates/installprop2xml.properties ---------------------------------------------------------------------- diff --git a/tagsync/conf/templates/installprop2xml.properties b/tagsync/conf/templates/installprop2xml.properties index 27d3203..aa0c568 100644 --- a/tagsync/conf/templates/installprop2xml.properties +++ b/tagsync/conf/templates/installprop2xml.properties @@ -36,8 +36,6 @@ TAG_SOURCE_FILE_ENABLED = ranger.tagsync.source.file TAG_SOURCE_FILE_FILENAME = ranger.tagsync.source.file.filename TAG_SOURCE_FILE_CHECK_INTERVAL_IN_MILLIS = ranger.tagsync.source.file.check.interval.millis -TAG_SOURCE_NONE_ENABLED = ranger.tagsync.source.none - TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = ranger.tagsync.atlas.to.ranger.service.mapping TAGSYNC_ATLAS_CUSTOM_RESOURCE_MAPPERS = ranger.tagsync.atlas.custom.resource.mappers http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/tagsync/conf/templates/ranger-tagsync-template.xml ---------------------------------------------------------------------- diff --git a/tagsync/conf/templates/ranger-tagsync-template.xml b/tagsync/conf/templates/ranger-tagsync-template.xml index 03bb3fa..3d656df 100644 --- a/tagsync/conf/templates/ranger-tagsync-template.xml +++ b/tagsync/conf/templates/ranger-tagsync-template.xml @@ -60,10 +60,6 @@ <value></value> </property> <property> - <name>ranger.tagsync.source.none</name> - <value></value> - </property> - <property> <name>ranger.tagsync.atlas.to.ranger.service.mapping</name> <value></value> </property> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/tagsync/scripts/setup.py ---------------------------------------------------------------------- diff --git a/tagsync/scripts/setup.py b/tagsync/scripts/setup.py index 4404cdc..cd91c07 100755 --- a/tagsync/scripts/setup.py +++ b/tagsync/scripts/setup.py @@ -49,7 +49,7 @@ log4jFileName = 'log4j.properties' install2xmlMapFileName = 'installprop2xml.properties' templateFileName = 'ranger-tagsync-template.xml' initdProgramName = 'ranger-tagsync' -atlasApplicationPropFileName = 'application.properties' +atlasApplicationPropFileName = 'atlas-application.properties' installTemplateDirName = join(installPropDirName,'templates') confDistDirName = join(installPropDirName, confDistBaseDirName) http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/tagsync/src/main/java/org/apache/ranger/tagsync/model/TagSource.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/model/TagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/model/TagSource.java index 01bc365..5ef6c57 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/model/TagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/model/TagSource.java @@ -24,8 +24,6 @@ import java.util.Properties; public interface TagSource { - String TAG_SOURCE_NONE = "None"; - boolean initialize(Properties properties); void setTagSink(TagSink sink); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java index 4436a77..ac4dcd0 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java @@ -279,7 +279,7 @@ public class TagSynchronizer { className = "org.apache.ranger.tagsync.source.atlas.AtlasTagSource"; } else if (tagSourceName.equals("atlasrest")) { className = "org.apache.ranger.tagsync.source.atlasrest.AtlasRESTTagSource"; - } else if (!tagSourceName.equalsIgnoreCase(TagSource.TAG_SOURCE_NONE)) { + } else { LOG.error("tagSource name doesn't have any class associated with it. tagSourceName=" + tagSourceName + ", propertyPrefix=" + propPrefix); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java index fb974a5..510732b 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java @@ -25,9 +25,9 @@ import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.IStruct; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.util.StringUtils; import org.apache.ranger.plugin.model.RangerServiceResource; import org.apache.ranger.plugin.model.RangerTag; import org.apache.ranger.plugin.model.RangerTagDef; @@ -39,6 +39,7 @@ import java.util.*; public class AtlasNotificationMapper { private static final Log LOG = LogFactory.getLog(AtlasNotificationMapper.class); + @SuppressWarnings("unchecked") public static ServiceTags processEntityNotification(EntityNotification entityNotification) { ServiceTags ret = null; @@ -47,13 +48,22 @@ public class AtlasNotificationMapper { try { IReferenceableInstance entity = entityNotification.getEntity(); - if (AtlasResourceMapperUtil.isEntityTypeHandled(entity.getTypeName())) { - AtlasEntityWithTraits entityWithTraits = new AtlasEntityWithTraits(entityNotification.getEntity(), entityNotification.getAllTraits()); - ret = buildServiceTags(entityWithTraits, null); - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("Ranger not interested in Entity Notification for entity-type " + entityNotification.getEntity().getTypeName()); + if (entity != null) { + if (AtlasResourceMapperUtil.isEntityTypeHandled(entity.getTypeName())) { + AtlasEntityWithTraits entityWithTraits = new AtlasEntityWithTraits(entityNotification.getEntity(), entityNotification.getAllTraits()); + if (entityNotification.getOperationType() == EntityNotification.OperationType.ENTITY_DELETE) { + // Special case for ENTITY_DELETE notifications + ret = buildServiceTagsForEntityDeleteNotification(entityWithTraits); + } else { + ret = buildServiceTags(entityWithTraits, null); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Ranger not interested in Entity Notification for entity-type " + entityNotification.getEntity().getTypeName()); + } } + } else { + LOG.error("EntityNotification contains NULL entity"); } } catch (Exception exception) { LOG.error("createServiceTags() failed!! ", exception); @@ -86,6 +96,7 @@ public class AtlasNotificationMapper { break; } case ENTITY_UPDATE: + case ENTITY_DELETE: case TRAIT_ADD: case TRAIT_DELETE: { ret = true; @@ -99,6 +110,30 @@ public class AtlasNotificationMapper { return ret; } + static private ServiceTags buildServiceTagsForEntityDeleteNotification(AtlasEntityWithTraits entityWithTraits) throws Exception { + final ServiceTags ret; + + IReferenceableInstance entity = entityWithTraits.getEntity(); + + String guid = entity.getId()._getId(); + if (StringUtils.isNotBlank(guid)) { + ret = new ServiceTags(); + RangerServiceResource serviceResource = new RangerServiceResource(); + serviceResource.setGuid(guid); + ret.getServiceResources().add(serviceResource); + } else { + ret = buildServiceTags(entityWithTraits, null); + // tag-definitions should NOT be deleted as part of service-resource delete + ret.setTagDefinitions(MapUtils.EMPTY_MAP); + // Ranger deletes tags associated with deleted service-resource + ret.setTags(MapUtils.EMPTY_MAP); + } + + ret.setOp(ServiceTags.OP_DELETE); + + return ret; + } + static private Map<String, ServiceTags> buildServiceTags(List<AtlasEntityWithTraits> entitiesWithTraits) throws Exception { Map<String, ServiceTags> ret = new HashMap<String, ServiceTags>(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java index 49d6f61..af19823 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java @@ -43,7 +43,7 @@ import java.util.*; public class AtlasTagSource extends AbstractTagSource { private static final Log LOG = LogFactory.getLog(AtlasTagSource.class); - public static final String TAGSYNC_ATLAS_PROPERTIES_FILE_NAME = "application.properties"; + public static final String TAGSYNC_ATLAS_PROPERTIES_FILE_NAME = "atlas-application.properties"; public static final String TAGSYNC_ATLAS_KAFKA_ENDPOINTS = "atlas.kafka.bootstrap.servers"; public static final String TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT = "atlas.kafka.zookeeper.connect"; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/tagsync/src/main/resources/ranger-tagsync-site.xml ---------------------------------------------------------------------- diff --git a/tagsync/src/main/resources/ranger-tagsync-site.xml b/tagsync/src/main/resources/ranger-tagsync-site.xml index fe300b4..aa2357a 100644 --- a/tagsync/src/main/resources/ranger-tagsync-site.xml +++ b/tagsync/src/main/resources/ranger-tagsync-site.xml @@ -62,10 +62,6 @@ <value>60000</value> </property> <property> - <name>ranger.tagsync.source.none</name> - <value>false</value> - </property> - <property> <name>ranger.tagsync.atlas.to.ranger.service.mapping</name> <value /> </property>
