Repository: incubator-ranger Updated Branches: refs/heads/master 3a363c530 -> 2867cc55e
RANGER-807: TagSync - fixed 'replace' to delete service-resources that are not in full-sync serviceTags Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/2867cc55 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/2867cc55 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/2867cc55 Branch: refs/heads/master Commit: 2867cc55e7f4a4923bc9e73c5b3854d2cfe7305f Parents: 3a363c5 Author: Abhay Kulkarni <[email protected]> Authored: Tue Apr 5 17:34:58 2016 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Sat Apr 9 01:25:07 2016 -0700 ---------------------------------------------------------------------- .../apache/ranger/plugin/store/TagStore.java | 4 ++ .../ranger/plugin/store/file/TagFileStore.java | 47 ++++++++++++++++++ .../java/org/apache/ranger/biz/TagDBStore.java | 40 +++++++++++++++ .../apache/ranger/db/XXServiceResourceDao.java | 12 +++++ .../ranger/rest/ServiceTagsProcessor.java | 51 ++++++++++++++++++-- .../resources/META-INF/jpa_named_queries.xml | 4 ++ .../source/atlas/AtlasNotificationMapper.java | 6 +++ 7 files changed, 161 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2867cc55/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java index b135423..3c5a43b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java @@ -85,12 +85,16 @@ public interface TagStore { void deleteServiceResource(Long id) throws Exception; + void deleteServiceResourceByGuid(String guid) throws Exception; + RangerServiceResource getServiceResource(Long id) throws Exception; RangerServiceResource getServiceResourceByGuid(String guid) throws Exception; List<RangerServiceResource> getServiceResourcesByService(String serviceName) throws Exception; + List<String> getServiceResourceGuidsByService(String serviceName) throws Exception; + RangerServiceResource getServiceResourceByServiceAndResourceSignature(String serviceName, String resourceSignature) throws Exception; List<RangerServiceResource> getServiceResources(SearchFilter filter) throws Exception; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2867cc55/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java index 5f22f0d..cc983a6 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java @@ -665,6 +665,26 @@ public class TagFileStore extends AbstractTagStore { } @Override + public void deleteServiceResourceByGuid(String guid) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagFileStore.deleteServiceResourceByGuid(" + guid + ")"); + } + + try { + RangerServiceResource resource = getServiceResourceByGuid(guid); + + deleteServiceResource(resource); + } catch (Exception excp) { + throw new Exception("failed to delete service-resource with GUID=" + guid, excp); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileStore.deleteServiceResourceByGuid(" + guid + ")"); + } + + } + + @Override public RangerServiceResource getServiceResource(Long id) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("==> TagFileStore.getServiceResource(" + id + ")"); @@ -732,6 +752,33 @@ public class TagFileStore extends AbstractTagStore { } @Override + public List<String> getServiceResourceGuidsByService(String serviceName) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagFileStore.getServiceResourceGuidsByService(" + serviceName + ")"); + } + + List<String> ret = null; + + if (StringUtils.isNotBlank(serviceName)) { + List<RangerServiceResource> serviceResources = this.getServiceResourcesByService(serviceName); + + if(CollectionUtils.isNotEmpty(serviceResources)) { + ret = new ArrayList<String>(serviceResources.size()); + + for(RangerServiceResource serviceResource : serviceResources) { + ret.add(serviceResource.getGuid()); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileStore.getServiceResourceGuidsByService(" + serviceName + "): count=" + (ret == null ? 0 : ret.size())); + } + + return ret; + } + + @Override public RangerServiceResource getServiceResourceByServiceAndResourceSignature(String serviceName, String resourceSignature) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("==> TagFileStore.getServiceResourceByServiceAndResourceSignature(" + serviceName + ", " + resourceSignature + ")"); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2867cc55/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java index 28d7bf6..0ec37f1 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java @@ -88,6 +88,7 @@ public class TagDBStore extends AbstractTagStore { @Autowired GUIDUtil guidUtil; + @Override public RangerTagDef createTagDef(RangerTagDef tagDef) throws Exception { if (LOG.isDebugEnabled()) { @@ -531,6 +532,24 @@ public class TagDBStore extends AbstractTagStore { } @Override + public void deleteServiceResourceByGuid(String guid) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagDBStore.deleteServiceResourceByGuid(" + guid + ")"); + } + + RangerServiceResource resource = getServiceResourceByGuid(guid); + + if(resource != null) { + deleteResourceForServiceResource(resource.getId()); + rangerServiceResourceService.delete(resource); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagDBStore.deleteServiceResourceByGuid(" + guid + ")"); + } + } + + @Override public RangerServiceResource getServiceResource(Long id) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("==> TagDBStore.getServiceResource(" + id + ")"); @@ -582,6 +601,27 @@ public class TagDBStore extends AbstractTagStore { } @Override + public List<String> getServiceResourceGuidsByService(String serviceName) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagDBStore.getServiceResourceGuidsByService(" + serviceName + ")"); + } + + List<String> ret = null; + + XXService service = daoManager.getXXService().findByName(serviceName); + + if (service != null) { + ret = daoManager.getXXServiceResource().findServiceResourceGuidsInServiceId(service.getId()); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagDBStore.getServiceResourceGuidsByService(" + serviceName + "): count=" + (ret == null ? 0 : ret.size())); + } + + return ret; + } + + @Override public RangerServiceResource getServiceResourceByServiceAndResourceSignature(String serviceName, String resourceSignature) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("==> TagDBStore.getServiceResourceByServiceAndResourceSignature(" + serviceName + ", " + resourceSignature + ")"); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2867cc55/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java index 9257aaa..0907e2f 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java @@ -83,4 +83,16 @@ public class XXServiceResourceDao extends BaseDao<XXServiceResource> { return new ArrayList<XXServiceResource>(); } } + + public List<String> findServiceResourceGuidsInServiceId(Long serviceId) { + if (serviceId == null) { + return new ArrayList<String>(); + } + try { + return getEntityManager().createNamedQuery("XXServiceResource.findServiceResourceGuidsInServiceId", String.class) + .setParameter("serviceId", serviceId).getResultList(); + } catch (NoResultException e) { + return new ArrayList<String>(); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2867cc55/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java index 7e6900e..cf07deb 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java @@ -165,9 +165,10 @@ public class ServiceTagsProcessor { RangerServiceResource resourceInStore = null; if (existing == null) { - resource.setResourceSignature(resourceSignature); + resource.setResourceSignature(resourceSignature); resourceInStore = tagStore.createServiceResource(resource); + } else if (StringUtils.isEmpty(resource.getServiceName()) || MapUtils.isEmpty(resource.getResourceElements())) { resourceInStore = existing; } else { @@ -499,16 +500,60 @@ public class ServiceTagsProcessor { } } - // Delete all tagdef, tag, serviceResource and tagResourceMaps and then add all objects in provided ServiceTagsids private void replace(ServiceTags serviceTags) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("==> ServiceTagsProcessor.replace()"); } - tagStore.deleteAllTagObjectsForService(serviceTags.getServiceName()); + // Delete those service-resources which are in ranger database but not in provided service-tags + + Map<String, RangerServiceResource> serviceResourcesInServiceTagsMap = new HashMap<String, RangerServiceResource>(); + + List<RangerServiceResource> serviceResourcesInServiceTags = serviceTags.getServiceResources(); + + for (RangerServiceResource rangerServiceResource : serviceResourcesInServiceTags) { + String guid = rangerServiceResource.getGuid(); + + if(serviceResourcesInServiceTagsMap.containsKey(guid)) { + LOG.warn("duplicate service-resource found: guid=" + guid); + } + + serviceResourcesInServiceTagsMap.put(guid, rangerServiceResource); + } + + List<String> serviceResourcesInDb = tagStore.getServiceResourceGuidsByService(serviceTags.getServiceName()); + + for (String dbServiceResourceGuid : serviceResourcesInDb) { + + if (! serviceResourcesInServiceTagsMap.containsKey(dbServiceResourceGuid)) { + + if (LOG.isDebugEnabled()) { + LOG.debug("Deleting serviceResource(guid=" + dbServiceResourceGuid + ") and its tag-associations..."); + } + + List<RangerTagResourceMap> tagResourceMaps = tagStore.getTagResourceMapsForResourceGuid(dbServiceResourceGuid); + + if (CollectionUtils.isNotEmpty(tagResourceMaps)) { + for (RangerTagResourceMap tagResourceMap : tagResourceMaps) { + tagStore.deleteTagResourceMap(tagResourceMap.getId()); + } + } + + tagStore.deleteServiceResourceByGuid(dbServiceResourceGuid); + } + + } + + // Add/update resources and other tag-model objects provided in service-tags addOrUpdate(serviceTags); + // All private tags at this point are associated with some service-resource and shared + // tags cannot be deleted as they belong to some other service. In any case, any tags that + // are not associated with service-resource will not be downloaded to plugin. + + // Tag-defs cannot be deleted as there may be a shared tag that it refers to it. + if (LOG.isDebugEnabled()) { LOG.debug("<== ServiceTagsProcessor.replace()"); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2867cc55/security-admin/src/main/resources/META-INF/jpa_named_queries.xml ---------------------------------------------------------------------- diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml index c70dcba..469a400 100644 --- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml +++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml @@ -967,6 +967,10 @@ <query>select obj from XXServiceResource obj where obj.serviceId = :serviceId and obj.resourceSignature = :resourceSignature</query> </named-query> + <named-query name="XXServiceResource.findServiceResourceGuidsInServiceId"> + <query>select obj.guid from XXServiceResource obj where obj.serviceId = :serviceId</query> + </named-query> + <!-- End <== JPA Queries for Tag Based Policies --> <named-query name="XXTrxLog.getMaxIdOfXXTrxLog"> <query>select max(obj.id) from XXTrxLog obj</query> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2867cc55/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java ---------------------------------------------------------------------- diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java index a9316b5..7dc487c 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java @@ -146,6 +146,12 @@ public class AtlasNotificationMapper { } } + if (MapUtils.isNotEmpty(ret)) { + for (Map.Entry<String, ServiceTags> entry : ret.entrySet()) { + ServiceTags serviceTags = entry.getValue(); + serviceTags.setOp(ServiceTags.OP_REPLACE); + } + } return ret; }
