Repository: incubator-ranger Updated Branches: refs/heads/master b935426d1 -> 3a36875e9
RANGER-1146: Policy engine optimization: dynamic reordering of policy evaluation order Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/3a36875e Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/3a36875e Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/3a36875e Branch: refs/heads/master Commit: 3a36875e96aa2f689edd71ab77a8c74a6355493a Parents: b935426 Author: Abhay Kulkarni <[email protected]> Authored: Sat Aug 6 13:08:06 2016 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Mon Aug 15 18:34:14 2016 -0700 ---------------------------------------------------------------------- .../plugin/policyengine/RangerAccessResult.java | 26 +++ .../plugin/policyengine/RangerPolicyEngine.java | 2 + .../policyengine/RangerPolicyEngineImpl.java | 170 +++++++++++++++++-- .../policyengine/RangerPolicyRepository.java | 104 +++++++++--- .../RangerAbstractPolicyEvaluator.java | 36 +++- .../RangerDefaultPolicyEvaluator.java | 7 +- .../RangerOptimizedPolicyEvaluator.java | 5 - .../policyevaluator/RangerPolicyEvaluator.java | 9 + .../ranger/plugin/service/RangerBasePlugin.java | 64 ++++++- .../ranger/plugin/util/PerfDataRecorder.java | 41 ++--- .../ranger/plugin/util/RangerPerfTracer.java | 9 + .../plugin/policyengine/TestPolicyEngine.java | 4 + .../ranger/policyengine/CommandLineParser.java | 11 +- .../ranger/policyengine/PerfTestEngine.java | 17 +- .../ranger/policyengine/PerfTestOptions.java | 9 +- .../RangerPolicyenginePerfTester.java | 32 ++-- .../test/resources/testdata/test_modules.txt | 3 +- ranger-tools/testdata/test_modules.txt | 4 +- 18 files changed, 465 insertions(+), 88 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java index 49d32b2..5543848 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java @@ -31,7 +31,9 @@ public class RangerAccessResult { private boolean isAllowed = false; private boolean isAuditedDetermined = false; private boolean isAudited = false; + private long auditPolicyId = -1; private long policyId = -1; + private long evaluatedPoliciesCount = 0; private String reason = null; public RangerAccessResult(final String serviceName, final RangerServiceDef serviceDef, final RangerAccessRequest request) { @@ -42,7 +44,9 @@ public class RangerAccessResult { this.isAllowed = false; this.isAuditedDetermined = false; this.isAudited = false; + this.auditPolicyId = -1; this.policyId = -1; + this.evaluatedPoliciesCount = 0; this.reason = null; } @@ -50,12 +54,15 @@ public class RangerAccessResult { this.isAccessDetermined = other.getIsAccessDetermined(); this.isAllowed = other.getIsAllowed(); this.policyId = other.getPolicyId(); + this.evaluatedPoliciesCount = other.evaluatedPoliciesCount; this.reason = other.getReason(); } public void setAuditResultFrom(final RangerAccessResult other) { this.isAuditedDetermined = other.getIsAuditedDetermined(); this.isAudited = other.getIsAudited(); + this.auditPolicyId = other.getAuditPolicyId(); + this.evaluatedPoliciesCount += other.getEvaluatedPoliciesCount(); } /** @@ -144,12 +151,29 @@ public class RangerAccessResult { } /** + * @return the auditPolicyId + */ + public long getAuditPolicyId() { + return auditPolicyId; + } + + public long getEvaluatedPoliciesCount() { return this.evaluatedPoliciesCount; } + /** * @param policyId the policyId to set */ public void setPolicyId(long policyId) { this.policyId = policyId; } + /** + * @param policyId the auditPolicyId to set + */ + public void setAuditPolicyId(long policyId) { + this.auditPolicyId = policyId; + } + + public void incrementEvaluatedPoliciesCount() { this.evaluatedPoliciesCount++; } + public int getServiceType() { int ret = -1; @@ -177,6 +201,8 @@ public class RangerAccessResult { sb.append("isAuditedDetermined={").append(isAuditedDetermined).append("} "); sb.append("isAudited={").append(isAudited).append("} "); sb.append("policyId={").append(policyId).append("} "); + sb.append("auditPolicyId={").append(auditPolicyId).append("} "); + sb.append("evaluatedPoliciesCount={").append(evaluatedPoliciesCount).append("} "); sb.append("reason={").append(reason).append("} "); sb.append("}"); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java index 3e69d03..a5e92da 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java @@ -69,6 +69,8 @@ public interface RangerPolicyEngine { RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest request); + void reorderPolicyEvaluators(); + boolean preCleanup(); void setUseForwardedIPAddress(boolean useForwardedIPAddress); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index 1b8433b..90da6af 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@ -20,6 +20,7 @@ package org.apache.ranger.plugin.policyengine; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -34,7 +35,13 @@ import org.apache.ranger.plugin.util.RangerAccessRequestUtil; import org.apache.ranger.plugin.util.RangerPerfTracer; import org.apache.ranger.plugin.util.ServicePolicies; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; public class RangerPolicyEngineImpl implements RangerPolicyEngine { private static final Log LOG = LogFactory.getLog(RangerPolicyEngineImpl.class); @@ -43,6 +50,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { private static final Log PERF_POLICYENGINE_REQUEST_LOG = RangerPerfTracer.getPerfLogger("policyengine.request"); private static final Log PERF_POLICYENGINE_AUDIT_LOG = RangerPerfTracer.getPerfLogger("policyengine.audit"); private static final Log PERF_CONTEXTENRICHER_REQUEST_LOG = RangerPerfTracer.getPerfLogger("contextenricher.request"); + private static final Log PERF_POLICYENGINE_REBALANCE_LOG = RangerPerfTracer.getPerfLogger("policyengine.rebalance"); + private static final Log PERF_POLICYENGINE_USAGE_LOG = RangerPerfTracer.getPerfLogger("policyengine.usage"); private static final int MAX_POLICIES_FOR_CACHE_TYPE_EVALUATOR = 100; @@ -51,6 +60,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { private List<RangerContextEnricher> allContextEnrichers; + private final Map<Long, RangerPolicyEvaluator> policyEvaluatorsMap; + private boolean useForwardedIPAddress = false; private String[] trustedProxyAddresses = null; @@ -132,6 +143,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { this.allContextEnrichers = tmpList; + policyEvaluatorsMap = createPolicyEvaluatorsMap(); + RangerPerfTracer.log(perf); if (PERF_POLICYENGINE_INIT_LOG.isDebugEnabled()) { @@ -160,6 +173,15 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { return policyRepository.getPolicyVersion(); } + public RangerPolicyEvaluator getPolicyEvaluator(Long id) { + return policyEvaluatorsMap.get(id); + } + + public RangerPolicy getPolicy(Long id) { + RangerPolicyEvaluator evaluator = getPolicyEvaluator(id); + return evaluator != null ? evaluator.getPolicy() : null; + } + @Override public RangerAccessResult createAccessResult(RangerAccessRequest request) { return new RangerAccessResult(this.getServiceName(), policyRepository.getServiceDef(), request); @@ -231,6 +253,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { RangerAccessResult ret = isAccessAllowedNoAudit(request); + updatePolicyUsageCounts(request, ret); + if (resultProcessor != null) { RangerPerfTracer perfAuditTracer = null; @@ -304,6 +328,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { ret.setIsAudited(false); } + updatePolicyUsageCounts(request, ret); + if (resultProcessor != null) { resultProcessor.processResult(ret); } @@ -339,6 +365,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { ret.setIsAudited(false); } + updatePolicyUsageCounts(request, ret); + if (resultProcessor != null) { resultProcessor.processResult(ret); } @@ -561,6 +589,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { List<RangerPolicyEvaluator> evaluators = policyRepository.getPolicyEvaluators(); for (RangerPolicyEvaluator evaluator : evaluators) { + ret.incrementEvaluatedPoliciesCount(); evaluator.evaluate(request, ret); if(ret.getIsAllowed() && !evaluator.hasDeny()) { // all policies having deny have been evaluated @@ -617,6 +646,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { tagEvalResult.setAuditResultFrom(result); for (RangerPolicyEvaluator evaluator : evaluators) { + tagEvalResult.incrementEvaluatedPoliciesCount(); if(! evaluator.isMatch(tagEvalRequest.getResource())) continue; @@ -641,6 +671,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { if (tagEvalResult.getIsAudited()) { result.setIsAudited(true); + result.setAuditPolicyId(tagEvalResult.getAuditPolicyId()); } if(!result.getIsAccessDetermined() && tagEvalResult.getIsAccessDetermined()) { @@ -667,25 +698,40 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { } } - private void setResourceServiceDef(RangerAccessRequest request) { - RangerAccessResource resource = request.getResource(); + @Override + public void reorderPolicyEvaluators() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> reorderPolicyEvaluators()"); + } + RangerPerfTracer perf = null; - if (resource.getServiceDef() == null) { - if (resource instanceof RangerMutableResource) { - RangerMutableResource mutable = (RangerMutableResource) resource; - mutable.setServiceDef(getServiceDef()); - } else { - LOG.debug("RangerPolicyEngineImpl.setResourceServiceDef(): Cannot set ServiceDef in RangerTagResourceMap."); + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_REBALANCE_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_REBALANCE_LOG, "RangerPolicyEngine.reorderPolicyEvaluators()"); + } + if (MapUtils.isNotEmpty(policyEvaluatorsMap)) { + for (Map.Entry<Long, RangerPolicyEvaluator> entry : policyEvaluatorsMap.entrySet()) { + entry.getValue().setUsageCountImmutable(); } } - } - private boolean hasTagPolicies() { - return tagPolicyRepository != null && CollectionUtils.isNotEmpty(tagPolicyRepository.getPolicies()); - } + if (tagPolicyRepository != null) { + tagPolicyRepository.reorderPolicyEvaluators(); + } + if (policyRepository != null) { + policyRepository.reorderPolicyEvaluators(); + } - private boolean hasResourcePolicies() { - return policyRepository != null && CollectionUtils.isNotEmpty(policyRepository.getPolicies()); + if (MapUtils.isNotEmpty(policyEvaluatorsMap)) { + for (Map.Entry<Long, RangerPolicyEvaluator> entry : policyEvaluatorsMap.entrySet()) { + entry.getValue().resetUsageCount(); + } + } + + RangerPerfTracer.log(perf); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== reorderPolicyEvaluators()"); + } } @Override @@ -783,6 +829,100 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { return sb; } + + + private void setResourceServiceDef(RangerAccessRequest request) { + RangerAccessResource resource = request.getResource(); + + if (resource.getServiceDef() == null) { + if (resource instanceof RangerMutableResource) { + RangerMutableResource mutable = (RangerMutableResource) resource; + mutable.setServiceDef(getServiceDef()); + } else { + LOG.debug("RangerPolicyEngineImpl.setResourceServiceDef(): Cannot set ServiceDef in RangerTagResourceMap."); + } + } + } + + private boolean hasTagPolicies() { + return tagPolicyRepository != null && CollectionUtils.isNotEmpty(tagPolicyRepository.getPolicies()); + } + + private boolean hasResourcePolicies() { + return policyRepository != null && CollectionUtils.isNotEmpty(policyRepository.getPolicies()); + } + + private Map<Long, RangerPolicyEvaluator> createPolicyEvaluatorsMap() { + Map<Long, RangerPolicyEvaluator> tmpPolicyEvaluatorMap = new HashMap<Long, RangerPolicyEvaluator>(); + + if (tagPolicyRepository != null) { + for (RangerPolicyEvaluator evaluator : tagPolicyRepository.getPolicyEvaluators()) { + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + } + for (RangerPolicyEvaluator evaluator : tagPolicyRepository.getDataMaskPolicyEvaluators()) { + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + } + for (RangerPolicyEvaluator evaluator : tagPolicyRepository.getRowFilterPolicyEvaluators()) { + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + } + } + for (RangerPolicyEvaluator evaluator : policyRepository.getPolicyEvaluators()) { + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + } + for (RangerPolicyEvaluator evaluator : policyRepository.getDataMaskPolicyEvaluators()) { + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + } + for (RangerPolicyEvaluator evaluator : policyRepository.getRowFilterPolicyEvaluators()) { + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + } + + return Collections.unmodifiableMap(tmpPolicyEvaluatorMap); + } + + private void updatePolicyUsageCounts(RangerAccessRequest accessRequest, RangerAccessResult accessResult) { + + boolean auditCountUpdated = false; + + if (accessResult.getIsAccessDetermined()) { + RangerPolicyEvaluator accessPolicy = getPolicyEvaluator(accessResult.getPolicyId()); + + if (accessPolicy != null) { + + if (accessPolicy.getPolicy().getIsAuditEnabled()) { + updateUsageCount(accessPolicy, 2); + accessResult.setAuditPolicyId(accessResult.getPolicyId()); + + auditCountUpdated = true; + } else { + updateUsageCount(accessPolicy, 1); + } + + } + } + + if (!auditCountUpdated && accessResult.getIsAuditedDetermined()) { + long auditPolicyId = accessResult.getAuditPolicyId(); + RangerPolicyEvaluator auditPolicy = auditPolicyId == -1 ? null : getPolicyEvaluator(auditPolicyId); + + updateUsageCount(auditPolicy, 1); + } + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_USAGE_LOG)) { + RangerAccessRequestImpl rangerAccessRequest = (RangerAccessRequestImpl) accessRequest; + RangerPerfTracer perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_USAGE_LOG, + "RangerPolicyEngine.usage(accessingUser=" + rangerAccessRequest.getUser() + + ",accessedResource=" + rangerAccessRequest.getResource().getAsString() + + ",accessType=" + rangerAccessRequest.getAccessType() + + ",evaluatedPoliciesCount=" + accessResult.getEvaluatedPoliciesCount() + ")"); + RangerPerfTracer.logAlways(perf); + } + } + + private void updateUsageCount(RangerPolicyEvaluator evaluator, int number) { + if (evaluator != null) { + evaluator.incrementUsageCount(number); + } + } } class RangerTagResource extends RangerAccessResourceImpl { private static final String KEY_TAG = "tag"; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index 01a547c..0012329 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -42,7 +42,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -public class RangerPolicyRepository { +class RangerPolicyRepository { private static final Log LOG = LogFactory.getLog(RangerPolicyRepository.class); private static final Log PERF_CONTEXTENRICHER_INIT_LOG = RangerPerfTracer.getPerfLogger("contextenricher.init"); @@ -51,6 +51,25 @@ public class RangerPolicyRepository { AUDIT_ALL, AUDIT_NONE, AUDIT_DEFAULT } + class AuditInfo { + final boolean isAudited; + final long auditPolicyId; + + AuditInfo() { + this(false, -1); + } + AuditInfo(boolean isAudited, long auditPolicyId) { + this.isAudited = isAudited; + this.auditPolicyId = auditPolicyId; + } + long getAuditPolicyId() { + return this.auditPolicyId; + } + boolean getIsAudited() { + return isAudited; + } + } + private final String serviceName; private final String appId; private final RangerServiceDef serviceDef; @@ -61,7 +80,7 @@ public class RangerPolicyRepository { private List<RangerPolicyEvaluator> dataMaskPolicyEvaluators; private List<RangerPolicyEvaluator> rowFilterPolicyEvaluators; private final AuditModeEnum auditModeEnum; - private final Map<String, Boolean> accessAuditCache; + private final Map<String, AuditInfo> accessAuditCache; private final String componentServiceName; private final RangerServiceDef componentServiceDef; @@ -98,7 +117,7 @@ public class RangerPolicyRepository { final int RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE = 64 * 1024; int auditResultCacheSize = RangerConfiguration.getInstance().getInt(propertyName, RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE); - accessAuditCache = Collections.synchronizedMap(new CacheMap<String, Boolean>(auditResultCacheSize)); + accessAuditCache = Collections.synchronizedMap(new CacheMap<String, AuditInfo>(auditResultCacheSize)); } else { accessAuditCache = null; } @@ -166,15 +185,15 @@ public class RangerPolicyRepository { public List<RangerContextEnricher> getContextEnrichers() { return contextEnrichers; } - public List<RangerPolicyEvaluator> getPolicyEvaluators() { + List<RangerPolicyEvaluator> getPolicyEvaluators() { return policyEvaluators; } - public List<RangerPolicyEvaluator> getDataMaskPolicyEvaluators() { + List<RangerPolicyEvaluator> getDataMaskPolicyEvaluators() { return dataMaskPolicyEvaluators; } - public List<RangerPolicyEvaluator> getRowFilterPolicyEvaluators() { + List<RangerPolicyEvaluator> getRowFilterPolicyEvaluators() { return rowFilterPolicyEvaluators; } @@ -323,10 +342,8 @@ public class RangerPolicyRepository { return policyItems; } - public static boolean isDelegateAdminPolicy(RangerPolicy policy) { - boolean ret = false; - - ret = hasDelegateAdminItems(policy.getPolicyItems()) + private static boolean isDelegateAdminPolicy(RangerPolicy policy) { + boolean ret = hasDelegateAdminItems(policy.getPolicyItems()) || hasDelegateAdminItems(policy.getDenyPolicyItems()) || hasDelegateAdminItems(policy.getAllowExceptions()) || hasDelegateAdminItems(policy.getDenyExceptions()); @@ -510,42 +527,49 @@ public class RangerPolicyRepository { LOG.debug("==> RangerPolicyRepository.setAuditEnabledFromCache()"); } - Boolean value = null; + final boolean auditResult; + final boolean foundInCache; switch (auditModeEnum) { case AUDIT_ALL: - value = Boolean.TRUE; + auditResult = true; + foundInCache = true; break; case AUDIT_NONE: - value = Boolean.FALSE; + auditResult = false; + foundInCache = true; break; default: - if (accessAuditCache != null) { - value = accessAuditCache.get(request.getResource().getAsString()); + AuditInfo auditInfo = accessAuditCache != null ? accessAuditCache.get(request.getResource().getAsString()) : null; + if (auditInfo != null) { + auditResult = auditInfo.getIsAudited(); + result.setAuditPolicyId(auditInfo.getAuditPolicyId()); + foundInCache = true; + } else { + auditResult = false; + foundInCache = false; } break; } - if ((value != null)) { - result.setIsAudited(value); + if (foundInCache) { + result.setIsAudited(auditResult); } if (LOG.isDebugEnabled()) { LOG.debug("<== RangerPolicyRepository.setAuditEnabledFromCache()"); } - return value != null; + return foundInCache; } - void storeAuditEnabledInCache(RangerAccessRequest request, RangerAccessResult ret) { + void storeAuditEnabledInCache(RangerAccessRequest request, RangerAccessResult result) { if (LOG.isDebugEnabled()) { LOG.debug("==> RangerPolicyRepository.storeAuditEnabledInCache()"); } - if (accessAuditCache != null && ret.getIsAuditedDetermined()) { - String strResource = request.getResource().getAsString(); - Boolean value = ret.getIsAudited() ? Boolean.TRUE : Boolean.FALSE; - accessAuditCache.put(strResource, value); + if (accessAuditCache != null && result.getIsAuditedDetermined()) { + accessAuditCache.put(request.getResource().getAsString(), new AuditInfo(result.getIsAudited(), result.getAuditPolicyId())); } if (LOG.isDebugEnabled()) { @@ -553,11 +577,7 @@ public class RangerPolicyRepository { } } - /** - * Remove nulls from policy resource values - * @param policy - */ - boolean scrubPolicy(RangerPolicy policy) { + private boolean scrubPolicy(RangerPolicy policy) { if (LOG.isDebugEnabled()) { LOG.debug("==> RangerPolicyRepository.scrubPolicy(" + policy + ")"); } @@ -584,6 +604,34 @@ public class RangerPolicyRepository { return altered; } + void reorderPolicyEvaluators() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> reorderPolicyEvaluators()"); + } + + this.policyEvaluators = getReorderedPolicyEvaluators(this.policyEvaluators); + this.dataMaskPolicyEvaluators = getReorderedPolicyEvaluators(this.dataMaskPolicyEvaluators); + this.rowFilterPolicyEvaluators = getReorderedPolicyEvaluators(this.rowFilterPolicyEvaluators); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== reorderPolicyEvaluators()"); + } + } + + private List<RangerPolicyEvaluator> getReorderedPolicyEvaluators(List<RangerPolicyEvaluator> evaluators) { + List<RangerPolicyEvaluator> ret = evaluators; + + if (CollectionUtils.isNotEmpty(evaluators)) { + + ret = new ArrayList<RangerPolicyEvaluator>(evaluators); + Collections.sort(ret); + + ret = Collections.unmodifiableList(ret); + } + + return ret; + } + @Override public String toString( ) { StringBuilder sb = new StringBuilder(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java index adc7d8c..8f45661 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java @@ -35,6 +35,8 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu private RangerPolicy policy = null; private RangerServiceDef serviceDef = null; private int evalOrder = 0; + protected long usageCount = 0; + protected boolean usageCountMutable = true; @Override @@ -74,10 +76,14 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu } @Override + public long getUsageCount() { + return usageCount; + } + + @Override public int getEvalOrder() { return evalOrder; } - @Override public boolean isAuditEnabled() { return policy != null && policy.getIsAuditEnabled(); @@ -89,7 +95,17 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu LOG.debug("==> RangerAbstractPolicyEvaluator.compareTo()"); } - int result = Integer.compare(this.getEvalOrder(), other.getEvalOrder()); + int result; + if (hasDeny() && !other.hasDeny()) { + result = -1; + } else if (!hasDeny() && other.hasDeny()) { + result = 1; + } else { + result = Long.compare(other.getUsageCount(), this.usageCount); + if (result == 0) { + result = Integer.compare(this.evalOrder, other.getEvalOrder()); + } + } if(LOG.isDebugEnabled()) { LOG.debug("<== RangerAbstractPolicyEvaluator.compareTo(), result:" + result); @@ -103,6 +119,22 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu } @Override + public void incrementUsageCount(int number) { + if (usageCountMutable) usageCount += number; + } + + @Override + public void setUsageCountImmutable() { + this.usageCountMutable = false; + } + + @Override + public void resetUsageCount() { + this.usageCount = 0; + this.usageCountMutable = true; + } + + @Override public String toString( ) { StringBuilder sb = new StringBuilder(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java index 368a695..f954ccf 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java @@ -177,6 +177,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator // We are done for determining if audit is needed for this policy if (isAuditEnabled()) { result.setIsAudited(true); + result.setAuditPolicyId(getPolicy().getId()); } } } @@ -249,6 +250,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator if (isResourceMatch || isResourceHeadMatch) { if (isAuditEnabled()) { result.setIsAudited(true); + result.setAuditPolicyId(getPolicy().getId()); } } } @@ -270,6 +272,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator evaluatePolicyItems(request, result); } } + } RangerPerfTracer.log(perf); @@ -314,6 +317,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator if (isResourceMatch || isResourceHeadMatch) { if (isAuditEnabled()) { result.setIsAudited(true); + result.setAuditPolicyId(getPolicy().getId()); } } } @@ -990,5 +994,6 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator } return ret; - } + } + } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java index 2e777ae..00b24d1 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java @@ -47,7 +47,6 @@ public class RangerOptimizedPolicyEvaluator extends RangerDefaultPolicyEvaluator private static final String RANGER_POLICY_EVAL_MATCH_ONE_CHARACTER_STRING = "?"; private static final int RANGER_POLICY_EVAL_SCORE_DEFAULT = 10000; - private static final int RANGER_POLICY_EVAL_SCORE_DISCOUNT_POLICY_HAS_DENY = 4000; private static final int RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_RESOURCE = 100; private static final int RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_USERSGROUPS = 25; @@ -202,10 +201,6 @@ public class RangerOptimizedPolicyEvaluator extends RangerDefaultPolicyEvaluator evalOrder -= customConditionsDiscount; } - if (hasDeny()) { - evalOrder -= RANGER_POLICY_EVAL_SCORE_DISCOUNT_POLICY_HAS_DENY; - } - if(LOG.isDebugEnabled()) { LOG.debug("<== RangerOptimizedPolicyEvaluator.computeEvalOrder(), policyName:" + policy.getName() + ", priority:" + evalOrder); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java index be97830..a7dc833 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java @@ -52,6 +52,14 @@ public interface RangerPolicyEvaluator extends Comparable<RangerPolicyEvaluator> int getEvalOrder(); + long getUsageCount(); + + void incrementUsageCount(int number); + + void setUsageCountImmutable(); + + void resetUsageCount(); + int getCustomConditionsCount(); boolean isAuditEnabled(); @@ -73,4 +81,5 @@ public interface RangerPolicyEvaluator extends Comparable<RangerPolicyEvaluator> boolean isAccessAllowed(Map<String, RangerPolicyResource> resources, String user, Set<String> userGroups, String accessType); void getResourceAccessInfo(RangerAccessRequest request, RangerResourceAccessInfo result); + } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index a000125..760c4a3 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -22,6 +22,8 @@ package org.apache.ranger.plugin.service; import java.util.Collection; import java.util.Hashtable; import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; @@ -30,7 +32,17 @@ import org.apache.ranger.admin.client.RangerAdminClient; import org.apache.ranger.admin.client.RangerAdminRESTClient; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.model.RangerServiceDef; -import org.apache.ranger.plugin.policyengine.*; +import org.apache.ranger.plugin.policyengine.RangerAccessRequest; +import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; +import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; +import org.apache.ranger.plugin.policyengine.RangerAccessResult; +import org.apache.ranger.plugin.policyengine.RangerAccessResultProcessor; +import org.apache.ranger.plugin.policyengine.RangerDataMaskResult; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngineImpl; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.ranger.plugin.policyengine.RangerResourceAccessInfo; +import org.apache.ranger.plugin.policyengine.RangerRowFilterResult; import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; import org.apache.ranger.plugin.util.GrantRevokeRequest; import org.apache.ranger.plugin.util.PolicyRefresher; @@ -51,6 +63,7 @@ public class RangerBasePlugin { private RangerAccessResultProcessor resultProcessor = null; private boolean useForwardedIPAddress = false; private String[] trustedProxyAddresses = null; + private Timer policyEngineRefreshTimer; Map<String, LogHistory> logHistoryList = new Hashtable<String, RangerBasePlugin.LogHistory>(); int logInterval = 30000; // 30 seconds @@ -126,6 +139,32 @@ public class RangerBasePlugin { refresher = new PolicyRefresher(this, serviceType, appId, serviceName, admin, pollingIntervalMs, cacheDir); refresher.setDaemon(true); refresher.startRefresher(); + + long policyReorderIntervalMs = RangerConfiguration.getInstance().getLong(propertyPrefix + ".policy.policyReorderInterval", 60 * 1000); + if (policyReorderIntervalMs >= 0 && policyReorderIntervalMs < 15 * 1000) { + policyReorderIntervalMs = 15 * 1000; + } + + if (LOG.isDebugEnabled()) { + LOG.debug(propertyPrefix + ".policy.policyReorderInterval:" + policyReorderIntervalMs); + } + + if (policyReorderIntervalMs > 0) { + policyEngineRefreshTimer = new Timer("PolicyEngineRefreshTimer", true); + try { + policyEngineRefreshTimer.schedule(new PolicyEngineRefresher(this), policyReorderIntervalMs, policyReorderIntervalMs); + if (LOG.isDebugEnabled()) { + LOG.debug("Scheduled PolicyEngineRefresher to reorder policies nbased on number of evaluations in and every " + policyReorderIntervalMs + " milliseconds"); + } + } catch (IllegalStateException exception) { + LOG.error("Error scheduling policyEngineRefresher:", exception); + LOG.error("*** PolicyEngine will NOT be reorderd based on number of evaluations every " + policyReorderIntervalMs + " milliseconds ***"); + policyEngineRefreshTimer = null; + } + } else { + LOG.info("Policies will NOT be reordered based on number of evaluations because " + + propertyPrefix + ".policy.policyReorderInterval is set to a negative number[" + policyReorderIntervalMs +"]"); + } } public void setPolicies(ServicePolicies policies) { @@ -154,17 +193,25 @@ public class RangerBasePlugin { RangerPolicyEngine policyEngine = this.policyEngine; + Timer policyEngineRefreshTimer = this.policyEngineRefreshTimer; + this.serviceName = null; this.policyEngine = null; this.refresher = null; + this.policyEngineRefreshTimer = null; if (refresher != null) { refresher.stopRefresher(); } + if (policyEngineRefreshTimer != null) { + policyEngineRefreshTimer.cancel(); + } + if (policyEngine != null) { policyEngine.cleanup(); } + } public void setResultProcessor(RangerAccessResultProcessor resultProcessor) { @@ -400,4 +447,19 @@ public class RangerBasePlugin { int counter=0; } + private class PolicyEngineRefresher extends TimerTask { + private final RangerBasePlugin plugin; + + PolicyEngineRefresher(RangerBasePlugin plugin) { + this.plugin = plugin; + } + + @Override + public void run() { + RangerPolicyEngine policyEngine = plugin.policyEngine; + if (policyEngine != null) { + policyEngine.reorderPolicyEvaluators(); + } + } + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java index ddc42c0..3fe96ac 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java @@ -36,7 +36,6 @@ public class PerfDataRecorder { private static volatile PerfDataRecorder instance = null; private Map<String, PerfStatistic> perfStatistics = new HashMap<String, PerfStatistic>(); - private boolean initPerfStatisticsOnce = true; public static void initialize(List<String> names) { if (instance == null) { @@ -57,6 +56,13 @@ public class PerfDataRecorder { instance.dumpStatistics(); } } + + public static void clearStatistics() { + if (instance != null) { + instance.clear(); + } + } + public static void recordStatistic(String tag, long elapsedTime) { if (instance != null) { instance.record(tag, elapsedTime); @@ -73,19 +79,16 @@ public class PerfDataRecorder { PerfStatistic perfStatistic = perfStatistics.get(tag); long averageTimeSpent = 0L; - long minTimeSpent = 0L; - long maxTimeSpent = 0L; + if (perfStatistic.numberOfInvocations.get() != 0L) { averageTimeSpent = perfStatistic.millisecondsSpent.get()/perfStatistic.numberOfInvocations.get(); - minTimeSpent = perfStatistic.minTimeSpent.get(); - maxTimeSpent = perfStatistic.maxTimeSpent.get(); } String logMsg = "[" + tag + "]" + - " execCount:" + perfStatistic.numberOfInvocations + - ", totalTimeTaken:" + perfStatistic.millisecondsSpent + - ", maxTimeTaken:" + maxTimeSpent + - ", minTimeTaken:" + minTimeSpent + + " execCount:" + perfStatistic.numberOfInvocations.get() + + ", totalTimeTaken:" + perfStatistic.millisecondsSpent.get() + + ", maxTimeTaken:" + perfStatistic.maxTimeSpent.get() + + ", minTimeTaken:" + perfStatistic.minTimeSpent.get() + ", avgTimeTaken:" + averageTimeSpent; LOG.info(logMsg); @@ -93,23 +96,25 @@ public class PerfDataRecorder { } } + private void clear() { + perfStatistics.clear(); + } + private void record(String tag, long elapsedTime) { PerfStatistic perfStatistic = perfStatistics.get(tag); - if (perfStatistic == null && !initPerfStatisticsOnce) { + if (perfStatistic == null) { synchronized (PerfDataRecorder.class) { perfStatistic = perfStatistics.get(tag); - if (perfStatistic == null) { + + if(perfStatistic == null) { perfStatistic = new PerfStatistic(); perfStatistics.put(tag, perfStatistic); } } } - if (perfStatistic != null) { - perfStatistic.addPerfDataItem(elapsedTime); - } - + perfStatistic.addPerfDataItem(elapsedTime); } private PerfDataRecorder(List<String> names) { @@ -118,8 +123,6 @@ public class PerfDataRecorder { // Create structure perfStatistics.put(name, new PerfStatistic()); } - } else { - initPerfStatisticsOnce = false; } } @@ -134,12 +137,12 @@ public class PerfDataRecorder { millisecondsSpent.getAndAdd(timeTaken); long min = minTimeSpent.get(); - if (timeTaken < min) { + if(timeTaken < min) { minTimeSpent.compareAndSet(min, timeTaken); } long max = maxTimeSpent.get(); - if (timeTaken > max) { + if(timeTaken > max) { maxTimeSpent.compareAndSet(max, timeTaken); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java index e130cc7..4b17110 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java @@ -71,6 +71,11 @@ public class RangerPerfTracer { } } + public static void logAlways(RangerPerfTracer tracer) { + if(tracer != null) { + tracer.logAlways(); + } + } public RangerPerfTracer(Log logger, String tag, String data) { this.logger = logger; this.tag = tag; @@ -96,4 +101,8 @@ public class RangerPerfTracer { logger.debug("[PERF] " + tag + data + ": " + elapsedTime); } } + public void logAlways() { + long elapsedTime = getElapsedTime(); + logger.debug("[PERF] " + tag + data + ": " + elapsedTime); + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java index 1d03ef8..ab5626e 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java @@ -300,11 +300,15 @@ public class TestPolicyEngine { policyEngine = new RangerPolicyEngineImpl(testName, servicePolicies, policyEngineOptions); policyEngine.setUseForwardedIPAddress(useForwardedIPAddress); policyEngine.setTrustedProxyAddresses(trustedProxyAddresses); + long requestCount = 0L; RangerAccessRequest request = null; for(TestData test : testCase.tests) { request = test.request; + if ((requestCount++ % 10) == 1) { + policyEngine.reorderPolicyEvaluators(); + } if (request.getContext().containsKey(RangerAccessRequestUtil.KEY_CONTEXT_TAGS) || request.getContext().containsKey(RangerAccessRequestUtil.KEY_CONTEXT_REQUESTED_RESOURCES)) { // Create a new AccessRequest http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java index ab7d532..d6c028f 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java @@ -52,6 +52,8 @@ public class CommandLineParser private int concurrentClientCount = 1; private int iterationsCount = 1; + private boolean isDynamicReorderingEnabled = false; + private Options options = new Options(); CommandLineParser() {} @@ -60,7 +62,7 @@ public class CommandLineParser PerfTestOptions ret = null; if (parseArguments(args) && validateInputFiles()) { // Instantiate a data-object and return - ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount); + ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount, isDynamicReorderingEnabled); } else { showUsage(); } @@ -75,6 +77,7 @@ public class CommandLineParser -r request-file-name-list -n number-of-iterations -p modules-to-collect-stats + -o If the concurrent-client-count is more than the number of files in the request-file-name-list, then reuse the request-file-names in a round-robin way @@ -94,6 +97,7 @@ public class CommandLineParser options.addOption("p", "statistics", true, "Modules for stat collection File Name"); options.addOption("c", "clients", true, "Number of concurrent clients"); options.addOption("n", "cycles", true, "Number of iterations"); + options.addOption("o", "optimize", false, "Enable usage-based policy reordering"); org.apache.commons.cli.CommandLineParser commandLineParser = new DefaultParser(); @@ -120,9 +124,14 @@ public class CommandLineParser if (iterationsOptionValue != null) { iterationsCount = Integer.parseInt(iterationsOptionValue); } + if (commandLine.hasOption("o")) { + isDynamicReorderingEnabled = true; + } + if (LOG.isDebugEnabled()) { LOG.debug("servicePoliciesFileName=" + servicePoliciesFileName + ", requestFileName=" + Arrays.toString(requestFileNames)); LOG.debug("concurrentClientCount=" + concurrentClientCount + ", iterationsCount=" + iterationsCount); + LOG.debug("isDynamicReorderingEnabled=" + isDynamicReorderingEnabled); } ret = true; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java index 38c6e2f..cf83260 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java @@ -24,6 +24,7 @@ import com.google.gson.GsonBuilder; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.plugin.policyengine.*; +import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; import org.apache.ranger.plugin.util.ServicePolicies; import java.io.InputStream; @@ -31,15 +32,20 @@ import java.io.InputStreamReader; import java.io.Reader; import java.net.URL; import java.nio.charset.Charset; +import java.util.concurrent.atomic.AtomicLong; public class PerfTestEngine { static final Log LOG = LogFactory.getLog(PerfTestEngine.class); + static private final long POLICY_ENGINE_REORDER_AFTER_PROCESSING_REQUESTS_COUNT = 100; private final URL servicePoliciesFileURL; + private final boolean isDynamicReorderingEnabled; private RangerPolicyEngine policyEvaluationEngine; + private AtomicLong requestCount = new AtomicLong(); - public PerfTestEngine(final URL servicePoliciesFileURL) { + public PerfTestEngine(final URL servicePoliciesFileURL, boolean isDynamicReorderingEnabled) { this.servicePoliciesFileURL = servicePoliciesFileURL; + this.isDynamicReorderingEnabled = isDynamicReorderingEnabled; } public boolean init() { @@ -66,9 +72,12 @@ public class PerfTestEngine { RangerPolicyEngineOptions engineOptions = new RangerPolicyEngineOptions(); engineOptions.disableTagPolicyEvaluation = false; + engineOptions.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; policyEvaluationEngine = new RangerPolicyEngineImpl("perf-test", servicePolicies, engineOptions); + requestCount.set(0L); + ret = true; } catch (Exception excp) { @@ -101,6 +110,12 @@ public class PerfTestEngine { if (policyEvaluationEngine != null) { + long processedRequestCount = requestCount.getAndIncrement(); + + if (isDynamicReorderingEnabled && (processedRequestCount % POLICY_ENGINE_REORDER_AFTER_PROCESSING_REQUESTS_COUNT) == 0) { + policyEvaluationEngine.reorderPolicyEvaluators(); + } + policyEvaluationEngine.preProcess(request); ret = policyEvaluationEngine.isAccessAllowed(request, null); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java index f30cbd7..321ee69 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java @@ -26,17 +26,19 @@ public class PerfTestOptions { private final URL servicePoliciesFileURL; private final URL[] requestFileURLs; private final URL statCollectionFileURL; + private final boolean isDynamicReorderingEnabled; private final int concurrentClientCount; private final int iterationsCount; - PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount) { + PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount, boolean isDynamicReorderingEnabled) { this.servicePoliciesFileURL = servicePoliciesFileURL; this.requestFileURLs = requestFileURLs; this.statCollectionFileURL = statCollectionFileURL; this.iterationsCount = iterationsCount; this.concurrentClientCount = concurrentClientCount; + this.isDynamicReorderingEnabled = isDynamicReorderingEnabled; } public URL getServicePoliciesFileURL() { @@ -57,4 +59,7 @@ public class PerfTestOptions { public int getIterationsCount() { return iterationsCount; - }} + } + + public boolean getIsDynamicReorderingEnabled() { return isDynamicReorderingEnabled; } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java ---------------------------------------------------------------------- diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java index e752c62..50432f7 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java +++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java @@ -54,7 +54,7 @@ public class RangerPolicyenginePerfTester { URL servicePoliciesFileURL = perfTestOptions.getServicePoliciesFileURL(); - PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL); + PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL, perfTestOptions.getIsDynamicReorderingEnabled()); if (!perfTestEngine.init()) { LOG.error("Error initializing test data. Existing..."); System.exit(1); @@ -63,6 +63,22 @@ public class RangerPolicyenginePerfTester { URL[] requestFileURLs = perfTestOptions.getRequestFileURLs(); int requestFilesCount = requestFileURLs.length; + // warm-up policy engine + LOG.error("Warming up.."); + try { + for(URL requestFileURL : requestFileURLs) { + PerfTestClient perfTestClient = new PerfTestClient(perfTestEngine, 0, requestFileURL, 1); + + perfTestClient.init(); + perfTestClient.run(); + } + } catch(Throwable t) { + LOG.error("Error during warmup", t); + } + LOG.error("Warmed up!"); + + PerfDataRecorder.clearStatistics(); + int clientsCount = perfTestOptions.getConcurrentClientCount(); List<PerfTestClient> perfTestClients = new ArrayList<PerfTestClient>(clientsCount); @@ -89,12 +105,9 @@ public class RangerPolicyenginePerfTester { long totalMemory = runtime.totalMemory(); long freeMemory = runtime.freeMemory(); - LOG.info("Initial Memory Statistics:"); - LOG.info("\t\tMaximum memory available for the process:\t" + runtime.maxMemory()); - LOG.info("\t\tInitial In-Use memory:\t\t\t\t" + (totalMemory-freeMemory)); - LOG.info("\t\tInitial Free memory:\t\t\t\t" + freeMemory); - LOG.info("\n\n"); + LOG.info("Memory stats: max-available=:" + runtime.maxMemory() + "; in-use=" + (totalMemory-freeMemory) + "; free=" + freeMemory); + LOG.info("Starting " + perfTestClients.size() + " clients.."); for (PerfTestClient client : perfTestClients) { try { client.start(); @@ -102,13 +115,13 @@ public class RangerPolicyenginePerfTester { LOG.error("Error in starting client: " + client.getName(), t); } } + LOG.info("Started " + perfTestClients.size() + " clients"); LOG.info("Waiting for " + perfTestClients.size() + " clients to finish up"); for (PerfTestClient client : perfTestClients) { while (client.isAlive()) { try { - LOG.info("Waiting for " + client.getName() + " to finish up."); client.join(1000); runtime.gc(); @@ -116,10 +129,7 @@ public class RangerPolicyenginePerfTester { totalMemory = runtime.totalMemory(); freeMemory = runtime.freeMemory(); - LOG.info("Memory Statistics:"); - LOG.info("\t\tCurrent In-Use memory:\t\t" + (totalMemory-freeMemory)); - LOG.info("\t\tCurrent Free memory:\t\t" + freeMemory); - LOG.info("\n\n"); + LOG.info("Memory stats: max-available=:" + runtime.maxMemory() + "; in-use=" + (totalMemory-freeMemory) + "; free=" + freeMemory); } catch (InterruptedException interruptedException) { LOG.error("PerfTestClient.join() was interrupted"); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/ranger-tools/src/test/resources/testdata/test_modules.txt ---------------------------------------------------------------------- diff --git a/ranger-tools/src/test/resources/testdata/test_modules.txt b/ranger-tools/src/test/resources/testdata/test_modules.txt index 8eb5746..a355ec8 100644 --- a/ranger-tools/src/test/resources/testdata/test_modules.txt +++ b/ranger-tools/src/test/resources/testdata/test_modules.txt @@ -21,4 +21,5 @@ RangerTagEnricher.setServiceTags RangerPolicyEngine.init RangerPolicyEngine.preProcess RangerPolicyEngine.isAccessAllowedNoAudit - +RangerPolicyEngine.reorderPolicyEvaluators +RangerPolicyEngine.usage http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3a36875e/ranger-tools/testdata/test_modules.txt ---------------------------------------------------------------------- diff --git a/ranger-tools/testdata/test_modules.txt b/ranger-tools/testdata/test_modules.txt index f317aaf..f3a2f82 100644 --- a/ranger-tools/testdata/test_modules.txt +++ b/ranger-tools/testdata/test_modules.txt @@ -20,6 +20,8 @@ PolicyRefresher.loadPolicy RangerPolicyEngine.init RangerPolicyEngine.cleanUp +RangerPolicyEngine.reorderPolicyEvaluators +RangerPolicyEngine.usage RangerContextEnricher.init RangerPolicyEvaluator.init RangerPolicyItemEvaluator.init @@ -58,4 +60,4 @@ ServiceREST.getPolicy ServiceREST.getPolicies ServiceREST.countPolicies ServiceREST.getServicePolicies -ServiceREST.getServicePoliciesIfUpdated \ No newline at end of file +ServiceREST.getServicePoliciesIfUpdated
