Repository: incubator-ranger Updated Branches: refs/heads/master 85d5658fc -> 87b6d4b59
RANGER-323: Policy evaluation optimization: cache results of resource-match in policy Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/87b6d4b5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/87b6d4b5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/87b6d4b5 Branch: refs/heads/master Commit: 87b6d4b59425cdaa319b6022453488ded10f4938 Parents: 85d5658 Author: Madhan Neethiraj <[email protected]> Authored: Fri Mar 20 19:03:59 2015 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Fri Mar 20 20:02:54 2015 -0700 ---------------------------------------------------------------------- .../RangerPolicyEvaluatorFacade.java | 24 +++- .../policyengine/RangerPolicyRepository.java | 4 +- .../RangerCachedPolicyEvaluator.java | 81 +++++++++++++ .../RangerResourceAccessCache.java | 39 ++++++ .../RangerResourceAccessCacheImpl.java | 118 +++++++++++++++++++ 5 files changed, 257 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/87b6d4b5/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEvaluatorFacade.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEvaluatorFacade.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEvaluatorFacade.java index 92dedba..755f553 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEvaluatorFacade.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEvaluatorFacade.java @@ -19,11 +19,14 @@ package org.apache.ranger.plugin.policyengine; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.conditionevaluator.RangerConditionEvaluator; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.policyevaluator.RangerCachedPolicyEvaluator; import org.apache.ranger.plugin.policyevaluator.RangerDefaultPolicyEvaluator; import org.apache.ranger.plugin.policyevaluator.RangerOptimizedPolicyEvaluator; import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; @@ -33,13 +36,19 @@ import java.util.Map; public class RangerPolicyEvaluatorFacade implements RangerPolicyEvaluator, Comparable<RangerPolicyEvaluatorFacade> { private static final Log LOG = LogFactory.getLog(RangerPolicyEvaluatorFacade.class); - RangerDefaultPolicyEvaluator delegate = null; - int computedPolicyEvalOrder = 0; + RangerDefaultPolicyEvaluator delegate = null; + int computedPolicyEvalOrder = 0; - RangerPolicyEvaluatorFacade(boolean useCachePolicyEngine) { + RangerPolicyEvaluatorFacade() { super(); - delegate = new RangerOptimizedPolicyEvaluator(); + String evaluatorType = RangerConfiguration.getInstance().get("ranger.policyengine.evaluator.type", "cached"); + + if(StringUtils.isEmpty(evaluatorType) || StringUtils.equalsIgnoreCase(evaluatorType, "cached")) { + delegate = new RangerCachedPolicyEvaluator(); + } else { + delegate = new RangerOptimizedPolicyEvaluator(); + } } RangerPolicyEvaluator getPolicyEvaluator() { @@ -78,12 +87,12 @@ public class RangerPolicyEvaluatorFacade implements RangerPolicyEvaluator, Compa @Override public boolean isMatch(RangerAccessResource resource) { - return false; + return delegate.isMatch(resource); } @Override public boolean isSingleAndExactMatch(RangerAccessResource resource) { - return false; + return delegate.isSingleAndExactMatch(resource); } @Override @@ -121,10 +130,13 @@ public class RangerPolicyEvaluatorFacade implements RangerPolicyEvaluator, Compa if(LOG.isDebugEnabled()) { LOG.debug("==> RangerPolicyEvaluatorFacade.computePolicyEvalOrder()"); } + int result = delegate.computePolicyEvalOrder(); + if(LOG.isDebugEnabled()) { LOG.debug("<==RangerPolicyEvaluatorFacade.computePolicyEvalOrder(), result:" + result); } + return result; } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/87b6d4b5/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index 4ed11c1..2e4a79e 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -42,7 +42,6 @@ public class RangerPolicyRepository { private List<RangerContextEnricher> contextEnrichers = null; private RangerServiceDef serviceDef = null; // Not used at this time - private boolean useCachePolicyEngine = false; private Map<String, Boolean> accessAuditCache = null; private static int RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE = 64*1024; @@ -149,7 +148,7 @@ public class RangerPolicyRepository { RangerPolicyEvaluatorFacade ret = null; - ret = new RangerPolicyEvaluatorFacade(useCachePolicyEngine); + ret = new RangerPolicyEvaluatorFacade(); ret.init(policy, serviceDef); if(LOG.isDebugEnabled()) { @@ -230,7 +229,6 @@ public class RangerPolicyRepository { } } } - sb.append("useCachePolicyEngine={").append(useCachePolicyEngine).append("} "); sb.append("} "); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/87b6d4b5/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java new file mode 100644 index 0000000..f4db52b --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.policyevaluator; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.policyengine.RangerAccessResource; + +public class RangerCachedPolicyEvaluator extends RangerOptimizedPolicyEvaluator { + private static final Log LOG = LogFactory.getLog(RangerCachedPolicyEvaluator.class); + + private RangerResourceAccessCache cache = null; + + @Override + public void init(RangerPolicy policy, RangerServiceDef serviceDef) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerCachedPolicyEvaluator.init()"); + } + + super.init(policy, serviceDef); + + cache = RangerResourceAccessCacheImpl.getInstance(serviceDef, policy); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerCachedPolicyEvaluator.init()"); + } + } + + @Override + public boolean isMatch(RangerAccessResource resource) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerCachedPolicyEvaluator.isMatch(" + resource + ")"); + } + + boolean result = false; + + // Check in the evaluator-owned cache for the match, if found return. else call super.isMatch(), add result to cache + RangerResourceAccessCache.LookupResult lookup = cache.lookup(resource); + + if (lookup != RangerResourceAccessCache.LookupResult.IN_NOTMATCHED_CACHE) { + // We dont know definitely that this previously not matched + if (lookup != RangerResourceAccessCache.LookupResult.IN_MATCHED_CACHE) { + result = super.isMatch(resource); + + // update the cache with the result of the match + if(result) { + cache.add(resource, RangerResourceAccessCache.CacheType.MATCHED_CACHE); + } else { + cache.add(resource, RangerResourceAccessCache.CacheType.NOTMATCHED_CACHE); + } + } else { + result = true; + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerCachedPolicyEvaluator.isMatch(" + resource + "): " + result); + } + + return result; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/87b6d4b5/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerResourceAccessCache.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerResourceAccessCache.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerResourceAccessCache.java new file mode 100644 index 0000000..55f3a04 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerResourceAccessCache.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.plugin.policyevaluator; + +import org.apache.ranger.plugin.policyengine.RangerAccessResource; + +public interface RangerResourceAccessCache { + public enum LookupResult { + IN_MATCHED_CACHE, + IN_NOTMATCHED_CACHE, + NOT_FOUND, + ERROR + } + + public enum CacheType { + MATCHED_CACHE, + NOTMATCHED_CACHE + } + + LookupResult lookup(RangerAccessResource resource); + + void add(RangerAccessResource resource, CacheType cacheType); +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/87b6d4b5/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerResourceAccessCacheImpl.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerResourceAccessCacheImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerResourceAccessCacheImpl.java new file mode 100644 index 0000000..3388361 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerResourceAccessCacheImpl.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ranger.plugin.policyevaluator; + + +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.policyengine.CacheMap; +import org.apache.ranger.plugin.policyengine.RangerAccessResource; + + +public class RangerResourceAccessCacheImpl implements RangerResourceAccessCache { + private static final Log LOG = LogFactory.getLog(RangerResourceAccessCacheImpl.class); + + public synchronized static RangerResourceAccessCache getInstance(RangerServiceDef serviceDef, RangerPolicy policy) { + return new RangerResourceAccessCacheImpl(serviceDef, policy); + } + + private RangerServiceDef serviceDef = null; + + private Map<String, String> matchedResourceCache = null; + private Map<String, String> notMatchedResourceCache = null; + + private RangerResourceAccessCacheImpl(RangerServiceDef serviceDef, RangerPolicy policy) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerResourceAccessCacheImpl.constructor(), policyName:" + policy.getName()); + } + + int matchedCacheSize = RangerConfiguration.getInstance().getInt("ranger.policyengine.matched.cached.count", 1000); + int notMatchedCacheSize = RangerConfiguration.getInstance().getInt("ranger.policyengine.not.matched.cached.count", matchedCacheSize * 10); + + this.serviceDef = serviceDef; + + matchedResourceCache = new CacheMap<String, String>(matchedCacheSize); + notMatchedResourceCache = new CacheMap<String, String>(notMatchedCacheSize); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerResourceAccessCacheImpl.constructor(), policyName:" + policy.getName()); + } + } + + @Override + public LookupResult lookup(RangerAccessResource resource) { + String strResource = resource.getAsString(serviceDef); + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerResourceAccessCacheImpl.lookup(" + strResource + ")"); + } + + LookupResult result = LookupResult.NOT_FOUND; + + try { + synchronized(this) { + if (matchedResourceCache.containsKey(strResource)) { + result = LookupResult.IN_MATCHED_CACHE; + } else if(notMatchedResourceCache.containsKey(strResource)) { + result = LookupResult.IN_NOTMATCHED_CACHE; + } + } + } catch (Exception exception) { + result = LookupResult.ERROR; + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerResourceAccessCacheImpl.lookup(" + strResource + "): " + result); + } + + return result; + } + + @Override + public void add(RangerAccessResource resource, CacheType cacheType) { + String strResource = resource.getAsString(serviceDef); + + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerResourceAccessCacheImpl.add(" + strResource + ", " + cacheType + ")"); + } + + synchronized(this) { + switch (cacheType) { + case MATCHED_CACHE: + matchedResourceCache.put(strResource, strResource); + break; + + case NOTMATCHED_CACHE: + notMatchedResourceCache.put(strResource, strResource); + break; + default: + break; + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerResourceAccessCacheImpl.add(" + strResource + ", " + cacheType + ")"); + } + } +}
