Repository: ranger Updated Branches: refs/heads/master 97280ee10 -> 33bf690cc
RANGER-1843: Tag enricher performance improvement in identifying tags for resource being accessed Project: http://git-wip-us.apache.org/repos/asf/ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/33bf690c Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/33bf690c Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/33bf690c Branch: refs/heads/master Commit: 33bf690cc7d09aacd10fa9f12ba073e5e6e1e124 Parents: 97280ee Author: Abhay Kulkarni <[email protected]> Authored: Wed Oct 18 12:28:06 2017 -0700 Committer: Abhay Kulkarni <[email protected]> Committed: Wed Oct 18 12:40:52 2017 -0700 ---------------------------------------------------------------------- .../contextenricher/RangerTagEnricher.java | 163 ++++++++++++++----- 1 file changed, 120 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ranger/blob/33bf690c/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java index 2b7a7b8..745bcf7 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java @@ -27,9 +27,11 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceResource; import org.apache.ranger.plugin.model.RangerTag; +import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; @@ -46,6 +48,7 @@ import java.io.FileWriter; import java.io.Reader; import java.io.Writer; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -61,6 +64,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { public static final String TAG_REFRESHER_POLLINGINTERVAL_OPTION = "tagRefresherPollingInterval"; public static final String TAG_RETRIEVER_CLASSNAME_OPTION = "tagRetrieverClassName"; public static final String TAG_DISABLE_TRIE_PREFILTER_OPTION = "disableTrieLookupPrefilter"; + public static final int[] allPolicyTypes = new int[] {RangerPolicy.POLICY_TYPE_ACCESS, RangerPolicy.POLICY_TYPE_DATAMASK, RangerPolicy.POLICY_TYPE_ROWFILTER}; private RangerTagRefresher tagRefresher; private RangerTagRetriever tagRetriever; @@ -148,6 +152,52 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { } } + /* + * This class implements a cache of result of look-up of keyset of policy-resources for each of the collections of hierarchies + * for policy types: access, datamask and rowfilter. If a keyset is examined for validity in a hierarchy of a policy-type, + * then that record is maintained in this cache for later look-up. + * + * The basic idea is that with a large number of tagged service-resources, this cache will speed up performance as well as put + * a cap on the upper bound because it is expected that the cardinality of set of all possible keysets for all resource-def + * combinations in a service-def will be much smaller than the number of service-resources. + */ + + static private class ResourceHierarchies { + private final Map<Collection<String>, Boolean> accessHierarchies = new HashMap<>(); + private final Map<Collection<String>, Boolean> dataMaskHierarchies = new HashMap<>(); + private final Map<Collection<String>, Boolean> rowFilterHierarchies = new HashMap<>(); + + public Boolean isValidHierarchy(int policyType, Collection<String> resourceKeys) { + switch (policyType) { + case RangerPolicy.POLICY_TYPE_ACCESS: + return accessHierarchies.get(resourceKeys); + case RangerPolicy.POLICY_TYPE_DATAMASK: + return dataMaskHierarchies.get(resourceKeys); + case RangerPolicy.POLICY_TYPE_ROWFILTER: + return rowFilterHierarchies.get(resourceKeys); + default: + return null; + } + } + + public void addHierarchy(int policyType, Collection<String> resourceKeys, Boolean isValid) { + switch (policyType) { + case RangerPolicy.POLICY_TYPE_ACCESS: + accessHierarchies.put(resourceKeys, isValid); + break; + case RangerPolicy.POLICY_TYPE_DATAMASK: + dataMaskHierarchies.put(resourceKeys, isValid); + break; + case RangerPolicy.POLICY_TYPE_ROWFILTER: + rowFilterHierarchies.put(resourceKeys, isValid); + break; + default: + LOG.error("unknown policy-type " + policyType); + break; + } + } + } + public void setServiceTags(final ServiceTags serviceTags) { if (serviceTags == null || CollectionUtils.isEmpty(serviceTags.getServiceResources())) { LOG.info("ServiceTags is null or there are no tagged resources for service " + serviceName); @@ -155,28 +205,51 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { } else { List<RangerServiceResourceMatcher> resourceMatchers = new ArrayList<>(); + RangerServiceDefHelper serviceDefHelper = new RangerServiceDefHelper(serviceDef, false); + List<RangerServiceResource> serviceResources = serviceTags.getServiceResources(); - if (CollectionUtils.isNotEmpty(serviceResources)) { - for (RangerServiceResource serviceResource : serviceResources) { - RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher(); + ResourceHierarchies hierarchies = new ResourceHierarchies(); + + for (RangerServiceResource serviceResource : serviceResources) { + final Collection<String> resourceKeys = serviceResource.getResourceElements().keySet(); - matcher.setServiceDef(this.serviceDef); - matcher.setPolicyResources(serviceResource.getResourceElements()); + for (int policyType : allPolicyTypes) { + Boolean isValidHierarchy = hierarchies.isValidHierarchy(policyType, resourceKeys); + if (isValidHierarchy == null) { // hierarchy not yet validated + isValidHierarchy = Boolean.FALSE; - if (LOG.isDebugEnabled()) { - LOG.debug("RangerTagEnricher.setServiceTags() - Initializing matcher with (resource=" + serviceResource - + ", serviceDef=" + this.serviceDef.getName() + ")"); + for (List<RangerServiceDef.RangerResourceDef> hierarchy : serviceDefHelper.getResourceHierarchies(policyType)) { + if (serviceDefHelper.hierarchyHasAllResources(hierarchy, resourceKeys)) { + isValidHierarchy = Boolean.TRUE; + + break; + } + } + hierarchies.addHierarchy(policyType, resourceKeys, isValidHierarchy); } - matcher.init(); - RangerServiceResourceMatcher serviceResourceMatcher = new RangerServiceResourceMatcher(serviceResource, matcher); - resourceMatchers.add(serviceResourceMatcher); - } + if (isValidHierarchy) { + RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher(); + + matcher.setServiceDef(this.serviceDef); + matcher.setPolicyResources(serviceResource.getResourceElements(), policyType); + + if (LOG.isDebugEnabled()) { + LOG.debug("RangerTagEnricher.setServiceTags() - Initializing matcher with (resource=" + serviceResource + + ", serviceDef=" + this.serviceDef.getName() + ")"); + } + matcher.init(); + + RangerServiceResourceMatcher serviceResourceMatcher = new RangerServiceResourceMatcher(serviceResource, matcher); + resourceMatchers.add(serviceResourceMatcher); + } + } } + Map<String, RangerResourceTrie<RangerServiceResourceMatcher>> serviceResourceTrie = null; if (!disableTrieLookupPrefilter) { @@ -284,18 +357,17 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { List<RangerServiceResourceMatcher> ret = null; - final List<RangerServiceResourceMatcher> serviceResourceMatchers = enrichedServiceTags.getServiceResourceMatchers(); final Map<String, RangerResourceTrie<RangerServiceResourceMatcher>> serviceResourceTrie = enrichedServiceTags.getServiceResourceTrie(); if (resource == null || resource.getKeys() == null || resource.getKeys().isEmpty() || serviceResourceTrie == null) { - ret = serviceResourceMatchers; + ret = enrichedServiceTags.getServiceResourceMatchers(); } else { Set<String> resourceKeys = resource.getKeys(); + List<List<RangerServiceResourceMatcher>> serviceResourceMatchersList = null; + List<RangerServiceResourceMatcher> smallestList = null; if (CollectionUtils.isNotEmpty(resourceKeys)) { - boolean isRetModifiable = false; - for (String resourceName : resourceKeys) { RangerResourceTrie<RangerServiceResourceMatcher> trie = serviceResourceTrie.get(resourceName); @@ -303,38 +375,43 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { continue; } - List<RangerServiceResourceMatcher> resourceEvaluators = trie.getEvaluatorsForResource(resource.getValue(resourceName)); - - if (CollectionUtils.isEmpty(resourceEvaluators)) { // no policies for this resource, bail out - ret = null; - } else if (ret == null) { // initialize ret with policies found for this resource - ret = resourceEvaluators; - } else { // remove matchers from ret that are not in resourceEvaluators - if (isRetModifiable) { - ret.retainAll(resourceEvaluators); - } else { - final List<RangerServiceResourceMatcher> shorterList; - final List<RangerServiceResourceMatcher> longerList; - - if (ret.size() < resourceEvaluators.size()) { - shorterList = ret; - longerList = resourceEvaluators; - } else { - shorterList = resourceEvaluators; - longerList = ret; - } + List<RangerServiceResourceMatcher> serviceResourceMatchers = trie.getEvaluatorsForResource(resource.getValue(resourceName)); - ret = new ArrayList<>(shorterList); - ret.retainAll(longerList); - isRetModifiable = true; - } + if (CollectionUtils.isEmpty(serviceResourceMatchers)) { // no policies for this resource, bail out + serviceResourceMatchersList = null; + smallestList = null; + break; } - if (CollectionUtils.isEmpty(ret)) { // if no matcher exists, bail out and return empty list - ret = null; - break; + if (smallestList == null) { + smallestList = serviceResourceMatchers; + } else { + if (serviceResourceMatchersList == null) { + serviceResourceMatchersList = new ArrayList<>(); + serviceResourceMatchersList.add(smallestList); + } + serviceResourceMatchersList.add(serviceResourceMatchers); + + if (smallestList.size() > serviceResourceMatchers.size()) { + smallestList = serviceResourceMatchers; + } } } + if (serviceResourceMatchersList != null) { + ret = new ArrayList<>(smallestList); + for (List<RangerServiceResourceMatcher> serviceResourceMatchers : serviceResourceMatchersList) { + if (serviceResourceMatchers != smallestList) { + // remove policies from ret that are not in serviceResourceMatchers + ret.retainAll(serviceResourceMatchers); + if (CollectionUtils.isEmpty(ret)) { // if no policy exists, bail out and return empty list + ret = null; + break; + } + } + } + } else { + ret = smallestList; + } } }
