Repository: incubator-ranger Updated Branches: refs/heads/ranger-0.5 087a7c859 -> 141018c5e
RANGER-1161: Policy evaluation optimization by using trie lookup to reduce number policies evaluated Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/141018c5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/141018c5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/141018c5 Branch: refs/heads/ranger-0.5 Commit: 141018c5e0870ee1e4b1a6c6b64592322262022d Parents: 087a7c8 Author: Madhan Neethiraj <[email protected]> Authored: Thu Aug 25 19:53:36 2016 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Thu Aug 25 19:53:36 2016 -0700 ---------------------------------------------------------------------- .../policyengine/RangerPolicyEngineImpl.java | 6 +- .../policyengine/RangerPolicyEngineOptions.java | 1 + .../policyengine/RangerPolicyRepository.java | 92 ++++- .../RangerDefaultPolicyEvaluator.java | 12 +- .../policyevaluator/RangerPolicyEvaluator.java | 6 + .../RangerDefaultPolicyResourceMatcher.java | 5 + .../RangerPolicyResourceMatcher.java | 3 + .../RangerAbstractResourceMatcher.java | 3 + .../resourcematcher/RangerResourceMatcher.java | 2 + .../ranger/plugin/service/RangerBasePlugin.java | 2 +- .../ranger/plugin/util/RangerResourceTrie.java | 391 +++++++++++++++++++ .../ranger/policyengine/CommandLineParser.java | 8 +- .../ranger/policyengine/PerfTestEngine.java | 8 +- .../ranger/policyengine/PerfTestOptions.java | 10 +- .../RangerPolicyenginePerfTester.java | 8 +- .../org/apache/ranger/rest/ServiceREST.java | 1 + 16 files changed, 543 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/141018c5/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index ae48a71..cc13550 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@ -255,7 +255,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { } boolean ret = false; - for(RangerPolicyEvaluator evaluator : policyRepository.getPolicyEvaluators()) { + for(RangerPolicyEvaluator evaluator : policyRepository.getPolicyEvaluators(resource)) { ret = evaluator.isAccessAllowed(resource, user, userGroups, accessType); if(ret) { @@ -361,7 +361,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { RangerResourceAccessInfo ret = new RangerResourceAccessInfo(request); - for(RangerPolicyEvaluator evaluator : policyRepository.getPolicyEvaluators()) { + for(RangerPolicyEvaluator evaluator : policyRepository.getPolicyEvaluators(request.getResource())) { evaluator.getResourceAccessInfo(request, ret); } @@ -380,7 +380,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { RangerAccessResult ret = createAccessResult(request); if(ret != null && request != null) { - List<RangerPolicyEvaluator> evaluators = policyRepository.getPolicyEvaluators(); + List<RangerPolicyEvaluator> evaluators = policyRepository.getPolicyEvaluators(request.getResource()); if(evaluators != null) { boolean foundInCache = policyRepository.setAuditEnabledFromCache(request, ret); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/141018c5/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java index 7cacfa8..98fdd6d 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java @@ -28,4 +28,5 @@ public class RangerPolicyEngineOptions { public boolean disableContextEnrichers = false; public boolean disableCustomConditions = false; public boolean evaluateDelegateAdminOnly = false; + public boolean disableTrieLookupPrefilter = false; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/141018c5/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index f522cfb..5de4583 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -28,17 +28,19 @@ import org.apache.ranger.plugin.contextenricher.RangerContextEnricher; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.policyevaluator.RangerCachedPolicyEvaluator; -import org.apache.ranger.plugin.policyevaluator.RangerDefaultPolicyEvaluator; import org.apache.ranger.plugin.policyevaluator.RangerOptimizedPolicyEvaluator; import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; import org.apache.ranger.plugin.util.RangerPerfTracer; +import org.apache.ranger.plugin.util.RangerResourceTrie; import org.apache.ranger.plugin.util.ServicePolicies; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; public class RangerPolicyRepository { private static final Log LOG = LogFactory.getLog(RangerPolicyRepository.class); @@ -52,6 +54,9 @@ public class RangerPolicyRepository { private List<RangerContextEnricher> contextEnrichers; private List<RangerPolicyEvaluator> policyEvaluators; private final Map<String, Boolean> accessAuditCache; + private final boolean disableTrieLookupPrefilter; + private final Map<String, RangerResourceTrie> policyResourceTrie; + private static int RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE = 64*1024; @@ -115,6 +120,16 @@ public class RangerPolicyRepository { } else { accessAuditCache = null; } + + this.disableTrieLookupPrefilter = options.disableTrieLookupPrefilter; + + if(this.disableTrieLookupPrefilter) { + policyResourceTrie = null; + } else { + policyResourceTrie = new HashMap<String, RangerResourceTrie>(); + } + + initResourceTries(); } public String getServiceName() { @@ -141,6 +156,10 @@ public class RangerPolicyRepository { return policyEvaluators; } + public List<RangerPolicyEvaluator> getPolicyEvaluators(RangerAccessResource resource) { + return disableTrieLookupPrefilter ? getPolicyEvaluators() : getPolicyEvaluators(policyResourceTrie, resource); + } + public static boolean isDelegateAdminPolicy(RangerPolicy policy) { boolean ret = false; @@ -237,6 +256,77 @@ public class RangerPolicyRepository { return ret; } + private void initResourceTries() { + if (!this.disableTrieLookupPrefilter) { + policyResourceTrie.clear(); + + if (serviceDef != null && serviceDef.getResources() != null) { + for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) { + policyResourceTrie.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, policyEvaluators)); + } + } + } + } + + private List<RangerPolicyEvaluator> getPolicyEvaluators(Map<String, RangerResourceTrie> resourceTrie, RangerAccessResource resource) { + List<RangerPolicyEvaluator> ret = null; + Set<String> resourceKeys = resource == null ? null : resource.getKeys(); + + if(CollectionUtils.isNotEmpty(resourceKeys)) { + boolean isRetModifiable = false; + + for(String resourceName : resourceKeys) { + RangerResourceTrie trie = resourceTrie.get(resourceName); + + if(trie == null) { // if no trie exists for this resource level, ignore and continue to next level + continue; + } + + List<RangerPolicyEvaluator> resourceEvaluators = trie.getPoliciesForResource(resource.getValue(resourceName)); + + if(CollectionUtils.isEmpty(resourceEvaluators)) { // no policies for this resource, bail out + ret = null; + } else if(ret == null) { // initialize ret with policies found for this resource + ret = resourceEvaluators; + } else { // remove policies from ret that are not in resourceEvaluators + if(isRetModifiable) { + ret.retainAll(resourceEvaluators); + } else { + final List<RangerPolicyEvaluator> shorterList; + final List<RangerPolicyEvaluator> longerList; + + if (ret.size() < resourceEvaluators.size()) { + shorterList = ret; + longerList = resourceEvaluators; + } else { + shorterList = resourceEvaluators; + longerList = ret; + } + + ret = new ArrayList<>(shorterList); + ret.retainAll(longerList); + isRetModifiable = true; + } + } + + if(CollectionUtils.isEmpty(ret)) { // if no policy exists, bail out and return empty list + ret = null; + break; + } + } + } + + if(ret == null) { + ret = Collections.emptyList(); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyRepository.getPolicyEvaluators(" + resource.getAsString() + "): evaluatorCount=" + ret.size()); + } + + return ret; + } + boolean setAuditEnabledFromCache(RangerAccessRequest request, RangerAccessResult result) { if (LOG.isDebugEnabled()) { LOG.debug("==> RangerPolicyRepository.setAuditEnabledFromCache()"); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/141018c5/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java index 67ea9b2..4385fec 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java @@ -45,9 +45,9 @@ import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; import org.apache.ranger.plugin.policyengine.RangerResourceAccessInfo; import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; import org.apache.ranger.plugin.util.RangerPerfTracer; - public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator { private static final Log LOG = LogFactory.getLog(RangerDefaultPolicyEvaluator.class); @@ -117,6 +117,16 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator } } + @Override + public RangerPolicyResourceMatcher getPolicyResourceMatcher() { + return resourceMatcher; + } + + @Override + public RangerResourceMatcher getResourceMatcher(String resourceName) { + return resourceMatcher != null ? resourceMatcher.getResourceMatcher(resourceName) : null; + } + @Override public void evaluate(RangerAccessRequest request, RangerAccessResult result) { if (LOG.isDebugEnabled()) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/141018c5/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java index a0f99e4..cd66dd6 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java @@ -31,6 +31,8 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResult; import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; import org.apache.ranger.plugin.policyengine.RangerResourceAccessInfo; +import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; public interface RangerPolicyEvaluator extends Comparable<RangerPolicyEvaluator> { @@ -50,6 +52,10 @@ public interface RangerPolicyEvaluator extends Comparable<RangerPolicyEvaluator> boolean isAuditEnabled(); + RangerPolicyResourceMatcher getPolicyResourceMatcher(); + + RangerResourceMatcher getResourceMatcher(String resourceName); + void evaluate(RangerAccessRequest request, RangerAccessResult result); boolean isMatch(RangerAccessResource resource); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/141018c5/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java index a020b68..5e0b54c 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java @@ -87,6 +87,11 @@ public class RangerDefaultPolicyResourceMatcher implements RangerPolicyResourceM } @Override + public RangerResourceMatcher getResourceMatcher(String resourceName) { + return matchers != null ? matchers.get(resourceName) : null; + } + + @Override public boolean isMatch(RangerAccessResource resource) { if(LOG.isDebugEnabled()) { LOG.debug("==> RangerDefaultPolicyResourceMatcher.isMatch(" + resource + ")"); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/141018c5/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java index 4a613a2..bcfc017 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.policyengine.RangerAccessResource; +import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; public interface RangerPolicyResourceMatcher { void setServiceDef(RangerServiceDef serviceDef); @@ -32,6 +33,8 @@ public interface RangerPolicyResourceMatcher { void init(); + RangerResourceMatcher getResourceMatcher(String resourceName); + boolean isMatch(RangerAccessResource resource); boolean isMatch(Map<String, RangerPolicyResource> resources); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/141018c5/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcher.java index 213d308..7614dac 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcher.java @@ -99,6 +99,9 @@ public abstract class RangerAbstractResourceMatcher implements RangerResourceMat } @Override + public boolean isMatchAny() { return isMatchAny; } + + @Override public boolean isSingleAndExactMatch(String resource) { if(LOG.isDebugEnabled()) { LOG.debug("==> RangerAbstractResourceMatcher.isSingleAndExactMatch(" + resource + ")"); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/141018c5/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerResourceMatcher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerResourceMatcher.java index 609d59d..4b0c17a 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerResourceMatcher.java @@ -29,6 +29,8 @@ public interface RangerResourceMatcher { void init(); + boolean isMatchAny(); + boolean isMatch(String resource); boolean isSingleAndExactMatch(String resource); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/141018c5/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index 501aa0a..0333ac2 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -98,7 +98,7 @@ public class RangerBasePlugin { policyEngineOptions.cacheAuditResults = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", true); policyEngineOptions.disableContextEnrichers = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", false); policyEngineOptions.disableCustomConditions = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", false); - + policyEngineOptions.disableTrieLookupPrefilter = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); RangerAdminClient admin = createAdminClient(propertyPrefix); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/141018c5/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java new file mode 100644 index 0000000..809c07e --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.util; + + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; +import org.apache.ranger.plugin.resourcematcher.RangerAbstractResourceMatcher; +import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +public class RangerResourceTrie { + private static final Log LOG = LogFactory.getLog(RangerResourceTrie.class); + + private static final String DEFAULT_WILDCARD_CHARS = "*?"; + + private final String resourceName; + private final boolean optIgnoreCase; + private final boolean optWildcard; + private final String wildcardChars; + private final TrieNode root; + + public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List<RangerPolicyEvaluator> evaluators) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ")"); + } + + Map<String, String> matcherOptions = resourceDef.getMatcherOptions(); + String strIgnoreCase = matcherOptions != null ? matcherOptions.get(RangerAbstractResourceMatcher.OPTION_IGNORE_CASE) : null; + String strWildcard = matcherOptions != null ? matcherOptions.get(RangerAbstractResourceMatcher.OPTION_WILD_CARD) : null; + + this.resourceName = resourceDef.getName(); + this.optIgnoreCase = strIgnoreCase != null ? Boolean.parseBoolean(strIgnoreCase) : false; + this.optWildcard = strWildcard != null ? Boolean.parseBoolean(strWildcard) : false;; + this.wildcardChars = optWildcard ? DEFAULT_WILDCARD_CHARS : ""; + this.root = new TrieNode(Character.valueOf((char)0)); + + for(RangerPolicyEvaluator evaluator : evaluators) { + RangerPolicy policy = evaluator.getPolicy(); + Map<String, RangerPolicyResource> policyResources = policy != null ? policy.getResources() : null; + RangerPolicyResource policyResource = policyResources != null ? policyResources.get(resourceName) : null; + + if(policyResource == null) { + continue; + } + + if(policyResource.getIsExcludes()) { + root.addWildcardPolicy(evaluator); + } else { + RangerResourceMatcher resourceMatcher = evaluator.getResourceMatcher(resourceName); + + if(resourceMatcher != null && resourceMatcher.isMatchAny()) { + root.addWildcardPolicy(evaluator); + } else { + for (String resource : policyResource.getValues()) { + insert(resource, policyResource.getIsRecursive(), evaluator); + } + } + } + } + + root.postSetup(); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + "): " + toString()); + } + } + + public String getResourceName() { + return resourceName; + } + + public List<RangerPolicyEvaluator> getPoliciesForResource(String resource) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerResourceTrie.getPoliciesForResource(" + resource + ")"); + } + + List<RangerPolicyEvaluator> ret = null; + + TrieNode curr = root; + + final int len = resource.length(); + for(int i = 0; i < len; i++) { + Character ch = getLookupChar(resource.charAt(i)); + TrieNode child = curr.getChild(ch); + + if(child == null) { + ret = curr.getWildcardPolicies(); + curr = null; // so that curr.getPolicies() will not be called below + break; + } + + curr = child; + } + + if(ret == null) { + if(curr != null) { + ret = curr.getPolicies(); + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerResourceTrie.getPoliciesForResource(" + resource + "): evaluatorCount=" + (ret == null ? 0 : ret.size())); + } + + return ret; + } + + public int getNodeCount() { + return root.getNodeCount(); + } + + public int getMaxDepth() { + return root.getMaxDepth(); + } + + public void reorderPolicyEvaluators() { + root.reorderPolicyEvaluators(); + } + + private Character getLookupChar(char ch) { + return optIgnoreCase ? Character.valueOf(Character.toLowerCase(ch)) : Character.valueOf(ch); + } + + private void insert(String resource, boolean isRecursive, RangerPolicyEvaluator evaluator) { + TrieNode curr = root; + boolean isWildcard = false; + + if(optIgnoreCase) { + resource = resource.toLowerCase(); + } + + final int len = resource.length(); + for(int i = 0; i < len; i++) { + Character ch = getLookupChar(resource.charAt(i)); + + if(optWildcard) { + if (wildcardChars.indexOf(ch) != -1) { + isWildcard = true; + break; + } + } + + curr = curr.getOrCreateChild(ch); + } + + if(isWildcard || isRecursive) { + curr.addWildcardPolicy(evaluator); + } else { + curr.addPolicy(evaluator); + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + sb.append("nodeCount=").append(getNodeCount()); + sb.append("; maxDepth=").append(getMaxDepth()); + sb.append(Character.LINE_SEPARATOR); + root.toString("", sb); + + return sb.toString(); + } +} + +class TrieNode { + private final Character c; + private Map<Character, TrieNode> children = null; + private List<RangerPolicyEvaluator> policies = null; + private List<RangerPolicyEvaluator> wildcardPolicies = null; + + TrieNode(Character c) { + this.c = c; + } + + Character getChar() { + return c; + } + + Map<Character, TrieNode> getChildren() { + return children; + } + + List<RangerPolicyEvaluator> getPolicies() { + return policies; + } + + List<RangerPolicyEvaluator> getWildcardPolicies() { + return wildcardPolicies; + } + + TrieNode getChild(Character c) { + TrieNode ret = children == null ? null : children.get(c); + + return ret; + } + + int getNodeCount() { + int ret = 1; + + if(children != null) { + for(Map.Entry<Character, TrieNode> entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + ret += child.getNodeCount(); + } + } + + return ret; + } + + int getMaxDepth() { + int ret = 0; + + if(children != null) { + for(Map.Entry<Character, TrieNode> entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + int maxChildDepth = child.getMaxDepth(); + + if(maxChildDepth > ret) { + ret = maxChildDepth; + } + } + } + + return ret + 1; + } + + TrieNode getOrCreateChild(Character c) { + if(children == null) { + children = new HashMap<Character, TrieNode>(); + } + + TrieNode child = children.get(c); + + if(child == null) { + child = new TrieNode(c); + children.put(c, child); + } + + return child; + } + + void addPolicy(RangerPolicyEvaluator evaluator) { + if(policies == null) { + policies = new ArrayList<RangerPolicyEvaluator>(); + } + + if(!policies.contains(evaluator)) { + policies.add(evaluator); + } + } + + void addPolicies(List<RangerPolicyEvaluator> evaluators) { + if(CollectionUtils.isNotEmpty(evaluators)) { + for(RangerPolicyEvaluator evaluator : evaluators) { + addPolicy(evaluator); + } + } + } + + void addWildcardPolicy(RangerPolicyEvaluator evaluator) { + if(wildcardPolicies == null) { + wildcardPolicies = new ArrayList<RangerPolicyEvaluator>(); + } + + if(!wildcardPolicies.contains(evaluator)) { + wildcardPolicies.add(evaluator); + } + } + + void addWildcardPolicies(List<RangerPolicyEvaluator> evaluators) { + if(CollectionUtils.isNotEmpty(evaluators)) { + for(RangerPolicyEvaluator evaluator : evaluators) { + addWildcardPolicy(evaluator); + } + } + } + + void postSetup() { + addPolicies(wildcardPolicies); + + if(wildcardPolicies != null) { + Collections.sort(wildcardPolicies); + } + + if(policies != null) { + Collections.sort(policies); + } + + if(children != null) { + for(Map.Entry<Character, TrieNode> entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + child.addWildcardPolicies(wildcardPolicies); + + child.postSetup(); + } + } + } + + void reorderPolicyEvaluators() { + wildcardPolicies = getSortedCopy(wildcardPolicies); + policies = getSortedCopy(policies); + } + + public void toString(String prefix, StringBuilder sb) { + String nodeValue = prefix; + + if(c != 0) { + nodeValue += c; + } + + sb.append("nodeValue=").append(nodeValue); + sb.append("; childCount=").append(children == null ? 0 : children.size()); + sb.append("; policies=[ "); + if(policies != null) { + for(RangerPolicyEvaluator evaluator : policies) { + sb.append(evaluator.getPolicy().getId()).append(" "); + } + } + sb.append("]"); + + sb.append("; wildcardPolicies=[ "); + if(wildcardPolicies != null) { + for(RangerPolicyEvaluator evaluator : wildcardPolicies) { + sb.append(evaluator.getPolicy().getId()).append(" "); + } + } + sb.append("]"); + sb.append(Character.LINE_SEPARATOR); + + if(children != null) { + for(Map.Entry<Character, TrieNode> entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + child.toString(nodeValue, sb); + } + } + } + + public void clear() { + children = null; + policies = null; + wildcardPolicies = null; + } + + private List<RangerPolicyEvaluator> getSortedCopy(List<RangerPolicyEvaluator> evaluators) { + final List<RangerPolicyEvaluator> ret; + + if(CollectionUtils.isNotEmpty(evaluators)) { + ret = new ArrayList<RangerPolicyEvaluator>(wildcardPolicies); + + Collections.sort(ret); + } else { + ret = evaluators; + } + + return ret; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/141018c5/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java index 0dc79a0..aa0a623 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java @@ -47,6 +47,7 @@ public class CommandLineParser private int concurrentClientCount = 1; private int iterationsCount = 1; + private boolean isTrieLookupPrefixDisabled = true; private Options options = new Options(); @@ -56,7 +57,7 @@ public class CommandLineParser PerfTestOptions ret = null; if (parseArguments(args) && validateInputFiles()) { // Instantiate a data-object and return - ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount); + ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount, isTrieLookupPrefixDisabled); } else { showUsage(-1); } @@ -90,6 +91,7 @@ public class CommandLineParser options.addOption("p", "statistics", true, "Modules for stat collection File Name"); options.addOption("c", "clients", true, "Number of concurrent clients"); options.addOption("n", "cycles", true, "Number of iterations"); + options.addOption("t", "trie-prefilter", false, "Enable trie-prefilter"); org.apache.commons.cli.CommandLineParser commandLineParser = new DefaultParser(); @@ -115,9 +117,13 @@ public class CommandLineParser if (iterationsOptionValue != null) { iterationsCount = Integer.parseInt(iterationsOptionValue); } + if (commandLine.hasOption("t")) { + isTrieLookupPrefixDisabled = false; + } if (LOG.isDebugEnabled()) { LOG.debug("servicePoliciesFileName=" + servicePoliciesFileName + ", requestFileName=" + Arrays.toString(requestFileNames)); LOG.debug("concurrentClientCount=" + concurrentClientCount + ", iterationsCount=" + iterationsCount); + LOG.debug("isTrieLookupPrefixDisabled=" + isTrieLookupPrefixDisabled); } ret = true; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/141018c5/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java index dfd8191..6e58f98 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java @@ -36,10 +36,12 @@ public class PerfTestEngine { static final Log LOG = LogFactory.getLog(PerfTestEngine.class); private final URL servicePoliciesFileURL; + private final RangerPolicyEngineOptions policyEngineOptions; private RangerPolicyEngine policyEvaluationEngine; - public PerfTestEngine(final URL servicePoliciesFileURL) { + public PerfTestEngine(final URL servicePoliciesFileURL, RangerPolicyEngineOptions policyEngineOptions) { this.servicePoliciesFileURL = servicePoliciesFileURL; + this.policyEngineOptions = policyEngineOptions; } public boolean init() { @@ -64,9 +66,7 @@ public class PerfTestEngine { servicePolicies = gsonBuilder.fromJson(reader, ServicePolicies.class); - RangerPolicyEngineOptions engineOptions = new RangerPolicyEngineOptions(); - - policyEvaluationEngine = new RangerPolicyEngineImpl(servicePolicies, engineOptions); + policyEvaluationEngine = new RangerPolicyEngineImpl(servicePolicies, policyEngineOptions); ret = true; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/141018c5/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java index f30cbd7..c1ab488 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java @@ -26,17 +26,18 @@ public class PerfTestOptions { private final URL servicePoliciesFileURL; private final URL[] requestFileURLs; private final URL statCollectionFileURL; - + private final boolean isTrieLookupPrefixDisabled; private final int concurrentClientCount; private final int iterationsCount; - PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount) { + PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount, boolean isTrieLookupPrefixDisabled) { this.servicePoliciesFileURL = servicePoliciesFileURL; this.requestFileURLs = requestFileURLs; this.statCollectionFileURL = statCollectionFileURL; this.iterationsCount = iterationsCount; this.concurrentClientCount = concurrentClientCount; + this.isTrieLookupPrefixDisabled = isTrieLookupPrefixDisabled; } public URL getServicePoliciesFileURL() { @@ -57,4 +58,7 @@ public class PerfTestOptions { public int getIterationsCount() { return iterationsCount; - }} + } + + public boolean getIsTrieLookupPrefixDisabled() { return isTrieLookupPrefixDisabled; } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/141018c5/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java index bcd1c68..22e14a1 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java @@ -21,6 +21,8 @@ package org.apache.ranger.policyengine; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; import org.apache.ranger.plugin.util.PerfDataRecorder; import java.io.BufferedReader; @@ -51,7 +53,11 @@ public class RangerPolicyenginePerfTester { URL servicePoliciesFileURL = perfTestOptions.getServicePoliciesFileURL(); - PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL); + RangerPolicyEngineOptions policyEngineOptions = new RangerPolicyEngineOptions(); + policyEngineOptions.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; + policyEngineOptions.disableTrieLookupPrefilter = perfTestOptions.getIsTrieLookupPrefixDisabled(); + + PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL, policyEngineOptions); if (!perfTestEngine.init()) { LOG.error("Error initializing test data. Existing..."); System.exit(1); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/141018c5/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 6de2bc1..0f5f5e9 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -1943,6 +1943,7 @@ public class ServiceREST { options.disableContextEnrichers = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true); options.disableCustomConditions = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true); options.evaluateDelegateAdminOnly = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.evaluate.delegateadmin.only", true); + options.disableTrieLookupPrefilter = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.trie.lookup.prefilter", false); RangerPolicyEngineCache.getInstance().setPolicyEngineOptions(options);; }
