RANGER-753: Optimized tag-download implementation. Instrumented policy download and policy evaluation for performance measurement.
Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/7a80c8e3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/7a80c8e3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/7a80c8e3 Branch: refs/heads/master Commit: 7a80c8e3522fb62ae5f3f53f6df786720a0569be Parents: 68ab77b Author: Abhay Kulkarni <[email protected]> Authored: Thu Nov 12 06:57:41 2015 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Tue Dec 1 21:14:37 2015 -0800 ---------------------------------------------------------------------- .../RangerAbstractConditionEvaluator.java | 3 + .../RangerConditionEvaluator.java | 1 - .../contextenricher/RangerTagEnricher.java | 50 +- .../ranger/plugin/model/RangerTagDef.java | 1 + .../policyengine/RangerPolicyEngineImpl.java | 62 +- .../policyengine/RangerPolicyRepository.java | 25 + .../RangerAbstractPolicyItemEvaluator.java | 9 +- .../RangerCachedPolicyEvaluator.java | 26 +- .../RangerDefaultPolicyEvaluator.java | 61 +- .../RangerDefaultPolicyItemEvaluator.java | 40 +- .../RangerOptimizedPolicyEvaluator.java | 12 +- .../RangerPolicyItemEvaluator.java | 2 + .../RangerAbstractResourceMatcher.java | 6 +- .../ranger/plugin/store/AbstractTagStore.java | 45 ++ .../apache/ranger/plugin/store/TagStore.java | 1 + .../ranger/plugin/util/PolicyRefresher.java | 11 +- .../plugin/policyengine/TestPolicyEngine.java | 2 +- .../src/test/resources/log4j.properties | 35 -- agents-common/src/test/resources/log4j.xml | 53 ++ security-admin/.gitignore | 2 - .../ranger/biz/RangerPolicyRetriever.java | 2 +- .../apache/ranger/biz/RangerTagDBRetriever.java | 597 +++++++++++++++++++ .../java/org/apache/ranger/biz/TagDBStore.java | 24 +- .../ranger/db/XXServiceResourceElementDao.java | 12 + .../db/XXServiceResourceElementValueDao.java | 25 + .../org/apache/ranger/db/XXTagAttributeDao.java | 23 + .../apache/ranger/db/XXTagAttributeDefDao.java | 23 + .../java/org/apache/ranger/db/XXTagDefDao.java | 13 + .../org/apache/ranger/rest/ServiceREST.java | 198 +++--- .../ranger/rest/ServiceTagsProcessor.java | 27 +- .../resources/META-INF/jpa_named_queries.xml | 77 ++- .../src/test/resources/log4j.properties | 35 -- security-admin/src/test/resources/log4j.xml | 53 ++ 33 files changed, 1287 insertions(+), 269 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerAbstractConditionEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerAbstractConditionEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerAbstractConditionEvaluator.java index 0bcb744..06263d1 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerAbstractConditionEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerAbstractConditionEvaluator.java @@ -47,4 +47,7 @@ public abstract class RangerAbstractConditionEvaluator implements RangerConditio @Override public void init() { } + + public RangerPolicyItemCondition getPolicyItemCondition() { return condition; } + } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerConditionEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerConditionEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerConditionEvaluator.java index 602b80e..9515000 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerConditionEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerConditionEvaluator.java @@ -31,7 +31,6 @@ public interface RangerConditionEvaluator { void setServiceDef(RangerServiceDef serviceDef); - void init(); boolean isMatched(RangerAccessRequest request); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java index b5662bf..e9fc42c 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java @@ -33,6 +33,7 @@ import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; import org.apache.ranger.plugin.util.RangerAccessRequestUtil; +import org.apache.ranger.plugin.util.RangerPerfTracer; import org.apache.ranger.plugin.util.ServiceTags; import java.io.*; @@ -43,6 +44,8 @@ import java.util.Map; public class RangerTagEnricher extends RangerAbstractContextEnricher { private static final Log LOG = LogFactory.getLog(RangerTagEnricher.class); + private static final Log PERF_ENRICHER_LOG = RangerPerfTracer.getPerfLogger("enricher"); + public static final String TAG_REFRESHER_POLLINGINTERVAL_OPTION = "tagRefresherPollingInterval"; public static final String TAG_RETRIEVER_CLASSNAME_OPTION = "tagRetrieverClassName"; @@ -51,8 +54,6 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { private RangerTagRetriever tagRetriever = null; - private long lastKnownVersion = -1L; - ServiceTags serviceTags = null; List<RangerServiceResourceMatcher> serviceResourceMatchers; @@ -100,7 +101,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { tagRetriever.setAppId(appId); tagRetriever.init(enricherDef.getEnricherOptions()); - tagRefresher = new RangerTagRefresher(tagRetriever, this, lastKnownVersion, cacheFile, pollingIntervalMs); + tagRefresher = new RangerTagRefresher(tagRetriever, this, -1L, cacheFile, pollingIntervalMs); try { tagRefresher.populateTags(); @@ -125,9 +126,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { LOG.debug("==> RangerTagEnricher.enrich(" + request + ")"); } - List<RangerServiceResourceMatcher> serviceResourceMatchersCopy = serviceResourceMatchers; - - List<RangerTag> matchedTags = findMatchingTags(request.getResource(), serviceResourceMatchersCopy); + List<RangerTag> matchedTags = findMatchingTags(request.getResource()); RangerAccessRequestUtil.setRequestTagsInContext(request.getContext(), matchedTags); @@ -137,15 +136,19 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { } public void setServiceTags(final ServiceTags serviceTags) { - this.serviceTags = serviceTags; - this.lastKnownVersion = serviceTags.getTagVersion(); List<RangerServiceResourceMatcher> resourceMatchers = new ArrayList<RangerServiceResourceMatcher>(); - List<RangerServiceResource> serviceResources = this.serviceTags.getServiceResources(); + List<RangerServiceResource> serviceResources = serviceTags.getServiceResources(); if (CollectionUtils.isNotEmpty(serviceResources)) { + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_ENRICHER_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_ENRICHER_LOG, "RangerTagEnricher.setServiceTags(serviceName=" + tagRetriever.getServiceName() + ",lastKnownVersion=" + serviceTags.getTagVersion() + ")"); + } + for (RangerServiceResource serviceResource : serviceResources) { RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher(); @@ -163,10 +166,12 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { resourceMatchers.add(serviceResourceMatcher); } - } - serviceResourceMatchers = resourceMatchers; + RangerPerfTracer.log(perf); + } + this.serviceResourceMatchers = resourceMatchers; + this.serviceTags = serviceTags; } @Override @@ -188,16 +193,19 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { return ret; } - private List<RangerTag> findMatchingTags(final RangerAccessResource resource, final List<RangerServiceResourceMatcher> resourceMatchers) { + private List<RangerTag> findMatchingTags(final RangerAccessResource resource) { if (LOG.isDebugEnabled()) { LOG.debug("==> RangerTagEnricher.findMatchingTags(" + resource + ")"); } List<RangerTag> ret = null; + final List<RangerServiceResourceMatcher> serviceResourceMatchers = this.serviceResourceMatchers; - if (CollectionUtils.isNotEmpty(resourceMatchers)) { + if (CollectionUtils.isNotEmpty(serviceResourceMatchers)) { - for (RangerServiceResourceMatcher resourceMatcher : resourceMatchers) { + final ServiceTags serviceTags = this.serviceTags; + + for (RangerServiceResourceMatcher resourceMatcher : serviceResourceMatchers) { boolean matchResult = resourceMatcher.isMatch(resource); @@ -226,14 +234,14 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { return ret; } - static private List<RangerTag> getTagsForServiceResource(ServiceTags serviceTags, RangerServiceResource serviceResource) { + static private List<RangerTag> getTagsForServiceResource(final ServiceTags serviceTags, final RangerServiceResource serviceResource) { List<RangerTag> ret = new ArrayList<RangerTag>(); - Long resourceId = serviceResource.getId(); + final Long resourceId = serviceResource.getId(); - Map<Long, List<Long>> resourceToTagIds = serviceTags.getResourceToTagIds(); - Map<Long, RangerTag> tags = serviceTags.getTags(); + final Map<Long, List<Long>> resourceToTagIds = serviceTags.getResourceToTagIds(); + final Map<Long, RangerTag> tags = serviceTags.getTags(); if (resourceId != null && MapUtils.isNotEmpty(resourceToTagIds) && MapUtils.isNotEmpty(tags)) { @@ -318,7 +326,11 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { if (tagEnricher != null) { ServiceTags serviceTags = null; + RangerPerfTracer perf = null; + if(RangerPerfTracer.isPerfTraceEnabled(PERF_ENRICHER_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_ENRICHER_LOG, "RangerTagRefresher.populateTags(serviceName=" + tagRetriever.getServiceName() + ",lastKnownVersion" + lastKnownVersion + ")"); + } serviceTags = tagRetriever.retrieveTags(lastKnownVersion); if (serviceTags == null) { @@ -329,6 +341,8 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { saveToCache(serviceTags); } + RangerPerfTracer.log(perf); + if (serviceTags != null) { tagEnricher.setServiceTags(serviceTags); LOG.info("RangerTagRefresher.populateTags() - Updated tags-cache to new version of tags, lastKnownVersion=" + lastKnownVersion + "; newVersion=" + serviceTags.getTagVersion()); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java index ba2a5d7..93f7b14 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java @@ -61,6 +61,7 @@ public class RangerTagDef extends RangerBaseModelObject { super(); setName(name); setSource(source); + setAttributeDefs(null); } public String getName() { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index 8adab7b..df6ca41 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@ -30,12 +30,15 @@ import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; import org.apache.ranger.plugin.util.RangerAccessRequestUtil; +import org.apache.ranger.plugin.util.RangerPerfTracer; import org.apache.ranger.plugin.util.ServicePolicies; import java.util.*; public class RangerPolicyEngineImpl implements RangerPolicyEngine { private static final Log LOG = LogFactory.getLog(RangerPolicyEngineImpl.class); + private static final Log PERF_POLICY_LOG = RangerPerfTracer.getPerfLogger("policy"); + private static final Log PERF_ENRICHER_LOG = RangerPerfTracer.getPerfLogger("enricher"); private final RangerPolicyRepository policyRepository; private final RangerPolicyRepository tagPolicyRepository; @@ -47,6 +50,12 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { LOG.debug("==> RangerPolicyEngineImpl(" + appId + ", " + servicePolicies + ", " + options + ")"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_LOG, "RangerPolicyEngine.init(appId=" + appId + ",hashCode=" + Integer.toHexString(System.identityHashCode(this)) + ")"); + } + if (options == null) { options = new RangerPolicyEngineOptions(); } @@ -90,6 +99,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { this.allContextEnrichers = tmpList; + RangerPerfTracer.log(perf); + if (LOG.isDebugEnabled()) { LOG.debug("<== RangerPolicyEngineImpl()"); } @@ -126,11 +137,21 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { List<RangerContextEnricher> enrichers = allContextEnrichers; if(!CollectionUtils.isEmpty(enrichers)) { + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_ENRICHER_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_ENRICHER_LOG, "RangerPolicyEngine.preProcess(requestHashCode=" + Integer.toHexString(System.identityHashCode(request)) + ")"); + } + for(RangerContextEnricher enricher : enrichers) { enricher.enrich(request); } + + RangerPerfTracer.log(perf); } + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerPolicyEngineImpl.preProcess(" + request + ")"); } @@ -168,6 +189,11 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { if (LOG.isDebugEnabled()) { LOG.debug("==> RangerPolicyEngineImpl.isAccessAllowed(" + request + ")"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_LOG, "RangerPolicyEngine.isAccessAllowed(requestHashCode=" + Integer.toHexString(System.identityHashCode(request)) + ")"); + } RangerAccessResult ret = isAccessAllowedNoAudit(request); @@ -175,6 +201,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { resultProcessor.processResult(ret); } + RangerPerfTracer.log(perf); + if (LOG.isDebugEnabled()) { LOG.debug("<== RangerPolicyEngineImpl.isAccessAllowed(" + request + "): " + ret); } @@ -216,6 +244,11 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { LOG.debug("==> RangerPolicyEngineImpl.isAccessAllowed(" + resource + ", " + user + ", " + userGroups + ", " + accessType + ")"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_LOG, "RangerPolicyEngine.isAccessAllowed(user=" + user + ",accessType=" + accessType + ")"); + } boolean ret = false; for (RangerPolicyEvaluator evaluator : policyRepository.getPolicyEvaluators()) { @@ -226,6 +259,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { } } + RangerPerfTracer.log(perf); + if (LOG.isDebugEnabled()) { LOG.debug("<== RangerPolicyEngineImpl.isAccessAllowed(" + resource + ", " + user + ", " + userGroups + ", " + accessType + "): " + ret); } @@ -311,6 +346,12 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { LOG.debug("==> RangerPolicyEngineImpl.isAccessAllowedNoAudit(" + request + ")"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_LOG, "RangerPolicyEngine.isAccessAllowedNoAudit(requestHashCode=" + Integer.toHexString(System.identityHashCode(request)) + ")"); + } + RangerAccessResult ret = createAccessResult(request); if (ret != null && request != null) { @@ -359,6 +400,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { } } + RangerPerfTracer.log(perf); + if (LOG.isDebugEnabled()) { LOG.debug("<== RangerPolicyEngineImpl.isAccessAllowedNoAudit(" + request + "): " + ret); } @@ -371,6 +414,12 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { LOG.debug("==> RangerPolicyEngineImpl.isAccessAllowedForTagPolicies(" + request + ", " + result + ")"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_LOG, "RangerPolicyEngine.isAccessAllowedForTagPolicies(requestHashCode=" + Integer.toHexString(System.identityHashCode(request)) + ")"); + } + List<RangerPolicyEvaluator> evaluators = tagPolicyRepository.getPolicyEvaluators(); if (CollectionUtils.isNotEmpty(evaluators)) { @@ -436,13 +485,11 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { break; // Break out of policy-evaluation loop } } - - if (LOG.isDebugEnabled()) { - LOG.debug("RangerPolicyEngineImpl.isAccessAllowedForTagPolicies() : result=" + result); - } } } + RangerPerfTracer.log(perf); + if (LOG.isDebugEnabled()) { LOG.debug("<== RangerPolicyEngineImpl.isAccessAllowedForTagPolicies(" + request + ", " + result + ")" ); } @@ -503,6 +550,11 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { LOG.debug("==> RangerPolicyEngineImpl.cleanup()"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_LOG, "RangerPolicyEngine.cleanUp(hashCode=" + Integer.toHexString(System.identityHashCode(this)) + ")"); + } preCleanup(); if (CollectionUtils.isNotEmpty(allContextEnrichers)) { @@ -513,6 +565,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { this.allContextEnrichers = null; + RangerPerfTracer.log(perf); + if (LOG.isDebugEnabled()) { LOG.debug("<== RangerPolicyEngineImpl.cleanup()"); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index 57b1b7d..0cde01a 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -31,12 +31,14 @@ import org.apache.ranger.plugin.policyevaluator.RangerCachedPolicyEvaluator; import org.apache.ranger.plugin.policyevaluator.RangerOptimizedPolicyEvaluator; import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; import org.apache.ranger.plugin.store.AbstractServiceStore; +import org.apache.ranger.plugin.util.RangerPerfTracer; import org.apache.ranger.plugin.util.ServicePolicies; import java.util.*; public class RangerPolicyRepository { private static final Log LOG = LogFactory.getLog(RangerPolicyRepository.class); + private static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("policy"); private final String serviceName; private final String appId; @@ -53,6 +55,12 @@ public class RangerPolicyRepository { RangerPolicyRepository(String appId, ServicePolicies servicePolicies, RangerPolicyEngineOptions options) { super(); + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerPolicyRepository.init(appId=" + appId + ",hashCode=" + Integer.toHexString(System.identityHashCode(this)) + ")"); + } + this.componentServiceName = this.serviceName = servicePolicies.getServiceName(); this.componentServiceDef = this.serviceDef = servicePolicies.getServiceDef(); @@ -78,6 +86,7 @@ public class RangerPolicyRepository { init(options); + RangerPerfTracer.log(perf); } RangerPolicyRepository(String appId, ServicePolicies.TagPolicies tagPolicies, RangerPolicyEngineOptions options, @@ -321,6 +330,12 @@ public class RangerPolicyRepository { RangerContextEnricher ret = null; + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerPolicyRepository.buildContextEnricher(name=" + enricherDef.getName() + ")"); + } + String name = enricherDef != null ? enricherDef.getName() : null; String clsName = enricherDef != null ? enricherDef.getEnricher() : null; @@ -343,6 +358,8 @@ public class RangerPolicyRepository { ret.init(); } + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerPolicyRepository.buildContextEnricher(" + enricherDef + "): " + ret); } @@ -356,6 +373,12 @@ public class RangerPolicyRepository { RangerPolicyEvaluator ret; + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerPolicyRepository.buildPolicyEvaluator(name=" + policy.getName() + ")"); + } + if(StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_DEFAULT)) { ret = new RangerOptimizedPolicyEvaluator(); } else if(StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED)) { @@ -366,6 +389,8 @@ public class RangerPolicyRepository { ret.init(policy, serviceDef, options); + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerPolicyRepository.buildPolicyEvaluator(" + policy + "," + serviceDef + "): " + ret); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyItemEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyItemEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyItemEvaluator.java index ffd1d79..d592182 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyItemEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyItemEvaluator.java @@ -47,15 +47,17 @@ public abstract class RangerAbstractPolicyItemEvaluator implements RangerPolicyI final RangerServiceDef serviceDef; final RangerPolicyItem policyItem; final int policyItemType; + final int policyItemIndex; final long policyId; final int evalOrder; List<RangerConditionEvaluator> conditionEvaluators = Collections.<RangerConditionEvaluator>emptyList(); - RangerAbstractPolicyItemEvaluator(RangerServiceDef serviceDef, RangerPolicy policy, RangerPolicyItem policyItem, int policyItemType, RangerPolicyEngineOptions options) { + RangerAbstractPolicyItemEvaluator(RangerServiceDef serviceDef, RangerPolicy policy, RangerPolicyItem policyItem, int policyItemType, int policyItemIndex, RangerPolicyEngineOptions options) { this.serviceDef = serviceDef; this.policyItem = policyItem; this.policyItemType = policyItemType; + this.policyItemIndex = policyItemIndex; this.options = options; this.policyId = policy != null && policy.getId() != null ? policy.getId() : -1; this.evalOrder = computeEvalOrder(); @@ -82,6 +84,11 @@ public abstract class RangerAbstractPolicyItemEvaluator implements RangerPolicyI } @Override + public int getPolicyItemIndex() { + return policyItemIndex; + } + + @Override public String getComments() { return policyItem == null ? null : policyItem.getComments(); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java index d67777c..580447b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerCachedPolicyEvaluator.java @@ -25,11 +25,14 @@ import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.ranger.plugin.util.RangerPerfTracer; public class RangerCachedPolicyEvaluator extends RangerOptimizedPolicyEvaluator { private static final Log LOG = LogFactory.getLog(RangerCachedPolicyEvaluator.class); + private static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("policy"); private RangerResourceAccessCache cache = null; + private String perfTag; @Override public void init(RangerPolicy policy, RangerServiceDef serviceDef, RangerPolicyEngineOptions options) { @@ -37,10 +40,23 @@ public class RangerCachedPolicyEvaluator extends RangerOptimizedPolicyEvaluator LOG.debug("==> RangerCachedPolicyEvaluator.init()"); } + StringBuffer perfTagBuffer = new StringBuffer(); + perfTagBuffer.append("policyId=").append(policy.getId()).append(",policyName=").append(policy.getName()); + + perfTag = perfTagBuffer.toString(); + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerCachedPolicyEvaluator.init(" + perfTag + ")"); + } + super.init(policy, serviceDef, options); cache = RangerResourceAccessCacheImpl.getInstance(serviceDef, policy); - + + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerCachedPolicyEvaluator.init()"); } @@ -54,6 +70,12 @@ public class RangerCachedPolicyEvaluator extends RangerOptimizedPolicyEvaluator boolean result = false; + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerCachedPolicyEvaluator.isMatch(" + perfTag + ",accessResource=" + resource.getAsString() + ")"); + } + // Check in the evaluator-owned cache for the match, if found return. else call super.isMatch(), add result to cache RangerResourceAccessCache.LookupResult lookup = cache.lookup(resource); @@ -73,6 +95,8 @@ public class RangerCachedPolicyEvaluator extends RangerOptimizedPolicyEvaluator } } + RangerPerfTracer.log(perf); + if (LOG.isDebugEnabled()) { LOG.debug("<== RangerCachedPolicyEvaluator.isMatch(" + resource + "): " + result); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java index da15c00..439b58d 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java @@ -44,10 +44,12 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResult; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.ranger.plugin.util.RangerPerfTracer; public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator { private static final Log LOG = LogFactory.getLog(RangerDefaultPolicyEvaluator.class); + private static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("policy"); private RangerPolicyResourceMatcher resourceMatcher = null; private List<RangerPolicyItemEvaluator> allowEvaluators = null; @@ -55,6 +57,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator private List<RangerPolicyItemEvaluator> allowExceptionEvaluators = null; private List<RangerPolicyItemEvaluator> denyExceptionEvaluators = null; private int customConditionsCount = 0; + private String perfTag; @Override public int getCustomConditionsCount() { @@ -67,6 +70,17 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator LOG.debug("==> RangerDefaultPolicyEvaluator.init()"); } + StringBuffer perfTagBuffer = new StringBuffer(); + perfTagBuffer.append("policyId=").append(policy.getId()).append(", policyName=").append(policy.getName()); + + perfTag = perfTagBuffer.toString(); + + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerDefaultPolicyEvaluator.init(" + perfTag + ")"); + } + super.init(policy, serviceDef, options); preprocessPolicy(policy, serviceDef); @@ -94,6 +108,8 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator Collections.sort(allowExceptionEvaluators); Collections.sort(denyExceptionEvaluators); + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerDefaultPolicyEvaluator.init()"); } @@ -105,6 +121,13 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator LOG.debug("==> RangerDefaultPolicyEvaluator.evaluate(" + request + ", " + result + ")"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerDefaultPolicyEvaluator.evaluate(requestHashCode=" + Integer.toHexString(System.identityHashCode(request)) + "," + + perfTag + ")"); + } + if (request != null && result != null) { boolean isResourceMatch = false; boolean isResourceHeadMatch = false; @@ -158,6 +181,8 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator } } + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerDefaultPolicyEvaluator.evaluate(" + request + ", " + result + ")"); } @@ -168,6 +193,12 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator LOG.debug("==> RangerDefaultPolicyEvaluator.evaluatePolicyItems(" + request + ", " + result + ", " + isResourceMatch + ")"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerDefaultPolicyEvaluator.evaluatePolicyItems(requestHashCode=" + Integer.toHexString(System.identityHashCode(request)) + "," + perfTag + ")"); + } + RangerPolicyItemEvaluator matchedPolicyItem = getMatchingPolicyItem(request, denyEvaluators, denyExceptionEvaluators); if(matchedPolicyItem == null && !result.getIsAllowed()) { // if not denied, evaluate allowItems only if not already allowed @@ -192,6 +223,8 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator } } + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerDefaultPolicyEvaluator.evaluatePolicyItems(" + request + ", " + result + ", " + isResourceMatch + ")"); } @@ -229,10 +262,18 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator boolean ret = false; + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerDefaultPolicyEvaluator.isMatch(resource=" + resource.getAsString() + "," + perfTag + ")"); + } + if(resourceMatcher != null) { ret = resourceMatcher.isMatch(resource); } + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerDefaultPolicyEvaluator.isMatch(" + resource + "): " + ret); } @@ -333,12 +374,20 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator boolean ret = false; + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerDefaultPolicyEvaluator.isAccessAllowed(hashCode=" + Integer.toHexString(System.identityHashCode(this)) + "," + perfTag + ")"); + } + RangerPolicyItemEvaluator item = this.getDeterminingPolicyItem(user, userGroups, accessType); if(item != null && item.getPolicyItemType() == RangerPolicyItemEvaluator.POLICY_ITEM_TYPE_ALLOW) { ret = true; } + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerDefaultPolicyEvaluator.isAccessAllowed(" + user + ", " + userGroups + ", " + accessType + "): " + ret); } @@ -464,8 +513,10 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator if(CollectionUtils.isNotEmpty(policyItems)) { ret = new ArrayList<RangerPolicyItemEvaluator>(); + int policyItemCounter = 1; + for(RangerPolicyItem policyItem : policyItems) { - RangerPolicyItemEvaluator itemEvaluator = new RangerDefaultPolicyItemEvaluator(serviceDef, policy, policyItem, policyItemType, options); + RangerPolicyItemEvaluator itemEvaluator = new RangerDefaultPolicyItemEvaluator(serviceDef, policy, policyItem, policyItemType, policyItemCounter++, options); itemEvaluator.init(); @@ -489,6 +540,12 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator RangerPolicyItemEvaluator ret = null; + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerDefaultPolicyEvaluator.getMatchingPolicyItem(requestHashCode=" + Integer.toHexString(System.identityHashCode(request)) + "," + perfTag + ")"); + } + if(CollectionUtils.isNotEmpty(evaluators)) { for (RangerPolicyItemEvaluator evaluator : evaluators) { if(evaluator.isMatch(request)) { @@ -513,6 +570,8 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator } } + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerDefaultPolicyEvaluator.getMatchingPolicyItem(" + request + "): " + ret); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyItemEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyItemEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyItemEvaluator.java index 39a0a5e..7f40bda 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyItemEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyItemEvaluator.java @@ -26,6 +26,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.conditionevaluator.RangerAbstractConditionEvaluator; import org.apache.ranger.plugin.conditionevaluator.RangerConditionEvaluator; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef; @@ -36,14 +37,18 @@ import org.apache.ranger.plugin.model.RangerServiceDef.RangerPolicyConditionDef; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.ranger.plugin.util.RangerPerfTracer; public class RangerDefaultPolicyItemEvaluator extends RangerAbstractPolicyItemEvaluator { private static final Log LOG = LogFactory.getLog(RangerDefaultPolicyItemEvaluator.class); + private static final Log PERF_POLICY_LOG = RangerPerfTracer.getPerfLogger("policy"); + private static final Log PERF_ITEM_LOG = RangerPerfTracer.getPerfLogger("item"); + private static final Log PERF_CONDITION_LOG = RangerPerfTracer.getPerfLogger("condition"); - public RangerDefaultPolicyItemEvaluator(RangerServiceDef serviceDef, RangerPolicy policy, RangerPolicyItem policyItem, int policyItemType, RangerPolicyEngineOptions options) { - super(serviceDef, policy, policyItem, policyItemType, options); + public RangerDefaultPolicyItemEvaluator(RangerServiceDef serviceDef, RangerPolicy policy, RangerPolicyItem policyItem, int policyItemType, int policyItemIndex, RangerPolicyEngineOptions options) { + super(serviceDef, policy, policyItem, policyItemType, policyItemIndex, options); } public void init() { @@ -54,6 +59,12 @@ public class RangerDefaultPolicyItemEvaluator extends RangerAbstractPolicyItemEv if (!getConditionsDisabledOption() && policyItem != null && CollectionUtils.isNotEmpty(policyItem.getConditions())) { conditionEvaluators = new ArrayList<RangerConditionEvaluator>(); + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_LOG, "RangerDefaultPolicyItemEvaluator.init(policyId=" + policyId + ",policyItemIndex=" + getPolicyItemIndex() + ")"); + } + for (RangerPolicyItemCondition condition : policyItem.getConditions()) { RangerPolicyConditionDef conditionDef = getConditionDef(condition.getType()); @@ -76,6 +87,7 @@ public class RangerDefaultPolicyItemEvaluator extends RangerAbstractPolicyItemEv LOG.error("RangerDefaultPolicyItemEvaluator(policyId=" + policyId + "): failed to instantiate condition evaluator '" + condition.getType() + "'; evaluatorClassName='" + conditionDef.getEvaluator() + "'"); } } + RangerPerfTracer.log(perf); } if(LOG.isDebugEnabled()) { @@ -90,6 +102,12 @@ public class RangerDefaultPolicyItemEvaluator extends RangerAbstractPolicyItemEv } boolean ret = false; + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_ITEM_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_ITEM_LOG, "RangerDefaultPolicyItemEvaluator.isMatch(policyId=" + policyId + ",policyItemIndex=" + getPolicyItemIndex() + ")"); + } + if(policyItem != null) { if(matchUserGroup(request.getUser(), request.getUserGroups())) { @@ -125,6 +143,8 @@ public class RangerDefaultPolicyItemEvaluator extends RangerAbstractPolicyItemEv } } + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerDefaultPolicyItemEvaluator.isMatch(" + request + "): " + ret); } @@ -217,8 +237,22 @@ public class RangerDefaultPolicyItemEvaluator extends RangerAbstractPolicyItemEv if(LOG.isDebugEnabled()) { LOG.debug("evaluating condition: " + conditionEvaluator); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_CONDITION_LOG)) { + String conditionType = null; + if (conditionEvaluator instanceof RangerAbstractConditionEvaluator) { + conditionType = ((RangerAbstractConditionEvaluator)conditionEvaluator).getPolicyItemCondition().getType(); + } + + perf = RangerPerfTracer.getPerfTracer(PERF_CONDITION_LOG, "RangerDefaultPolicyItemEvaluator.matchCustomConditions(policyId=" + policyId + ",policyItemIndex=" + getPolicyItemIndex() + ",policyConditionType=" + conditionType + ")"); + } + + boolean conditionEvalResult = conditionEvaluator.isMatched(request); + + RangerPerfTracer.log(perf); - if(!conditionEvaluator.isMatched(request)) { + if (!conditionEvalResult) { if(LOG.isDebugEnabled()) { LOG.debug(conditionEvaluator + " returned false"); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java index e81280f..8cd854f 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java @@ -29,12 +29,14 @@ import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.policyengine.RangerAccessResult; import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.ranger.plugin.util.RangerPerfTracer; import java.util.*; import java.lang.Math; public class RangerOptimizedPolicyEvaluator extends RangerDefaultPolicyEvaluator { private static final Log LOG = LogFactory.getLog(RangerOptimizedPolicyEvaluator.class); + private static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("policy"); private Set<String> groups = new HashSet<String>(); private Set<String> users = new HashSet<String>(); @@ -70,6 +72,12 @@ public class RangerOptimizedPolicyEvaluator extends RangerDefaultPolicyEvaluator LOG.debug("==> RangerOptimizedPolicyEvaluator.init()"); } + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerOptimizedPolicyEvaluator.init(policyId=" + policy.getId() + ",policyName=" + policy.getName() + ")"); + } + super.init(policy, serviceDef, options); preprocessPolicyItems(policy.getPolicyItems()); @@ -87,6 +95,8 @@ public class RangerOptimizedPolicyEvaluator extends RangerDefaultPolicyEvaluator setEvalOrder(computeEvalOrder()); + RangerPerfTracer.log(perf); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerOptimizedPolicyEvaluator.init()"); } @@ -222,7 +232,7 @@ public class RangerOptimizedPolicyEvaluator extends RangerDefaultPolicyEvaluator boolean ret = false; - if (hasPublicGroup || users.contains(user) || CollectionUtils.containsAny(groups, userGroups)) { + if (hasPublicGroup || users.contains(user) || CollectionUtils.containsAny(groups, userGroups)) { if (StringUtils.isEmpty(accessType)) { accessType = RangerPolicyEngine.ANY_ACCESS; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyItemEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyItemEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyItemEvaluator.java index 95d2b4e..1dfc8cf 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyItemEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyItemEvaluator.java @@ -37,6 +37,8 @@ public interface RangerPolicyItemEvaluator extends Comparable<RangerPolicyItemEv int getPolicyItemType(); + int getPolicyItemIndex(); + String getComments(); List<RangerConditionEvaluator> getConditionEvaluators(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcher.java index 56c4cfb..fd5133f 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerAbstractResourceMatcher.java @@ -35,6 +35,7 @@ public abstract class RangerAbstractResourceMatcher implements RangerResourceMat private static final Log LOG = LogFactory.getLog(RangerAbstractResourceMatcher.class); public final static String WILDCARD_ASTERISK = "*"; + public final static String WILDCARDS = "*?"; public final static String OPTIONS_SEP = ";"; public final static String OPTION_NV_SEP = "="; @@ -74,6 +75,7 @@ public abstract class RangerAbstractResourceMatcher implements RangerResourceMat policyIsExcludes = policyResource == null ? false : policyResource.getIsExcludes(); if(policyResource != null && policyResource.getValues() != null) { + boolean isWildCardPresent = !optWildCard; for(String policyValue : policyResource.getValues()) { if(StringUtils.isEmpty(policyValue)) { continue; @@ -81,10 +83,12 @@ public abstract class RangerAbstractResourceMatcher implements RangerResourceMat if(StringUtils.containsOnly(policyValue, WILDCARD_ASTERISK)) { isMatchAny = true; + } else if (!isWildCardPresent && StringUtils.containsAny(policyValue, WILDCARDS)) { + isWildCardPresent = true; } - policyValues.add(policyValue); } + optWildCard = optWildCard && isWildCardPresent; } if(policyValues.isEmpty()) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java index ed1b64d..f22a87a 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java @@ -22,6 +22,7 @@ package org.apache.ranger.plugin.store; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.plugin.model.*; +import org.apache.ranger.plugin.util.SearchFilter; import java.util.*; @@ -102,6 +103,50 @@ public abstract class AbstractTagStore implements TagStore { } return ret; } + + @Override + public void deleteAllTagObjectsForService(String serviceName, boolean isResourePrivateTag) throws Exception { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> AbstractTagStore.deleteAllTagObjectsForService(serviceName=" + serviceName + ", isResourcePrivateTag=" + isResourePrivateTag + ")"); + } + + List<RangerServiceResource> serviceResources = getServiceResourcesByService(serviceName); + + Set<Long> tagsToDelete = new HashSet<Long>(); + + + for (RangerServiceResource serviceResource : serviceResources) { + Long resourceId = serviceResource.getId(); + + List<RangerTagResourceMap> tagResourceMapsForService = getTagResourceMapsForResourceId(resourceId); + + if (isResourePrivateTag) { + for (RangerTagResourceMap tagResourceMap : tagResourceMapsForService) { + Long tagId = tagResourceMap.getTagId(); + RangerTag tag = getTag(tagId); + tagsToDelete.add(tag.getId()); + } + } + for (RangerTagResourceMap tagResourceMap : tagResourceMapsForService) { + deleteTagResourceMap(tagResourceMap.getId()); + } + } + + for (RangerServiceResource serviceResource : serviceResources) { + deleteServiceResource(serviceResource.getId()); + } + + for (Long tagId : tagsToDelete) { + deleteTag(tagId); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== AbstractTagStore.deleteAllTagObjectsForService(serviceName=" + serviceName + ", isResourcePrivateTag=" + isResourePrivateTag + ")"); + } + + } + } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java index ed20c51..104459d 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java @@ -124,4 +124,5 @@ public interface TagStore { ServiceTags getServiceTagsIfUpdated(String serviceName, Long lastKnownVersion) throws Exception; + void deleteAllTagObjectsForService(String serviceName, boolean isResourePrivateTag) throws Exception; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java index 0729339..58d99bb 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java @@ -37,6 +37,7 @@ import com.google.gson.GsonBuilder; public class PolicyRefresher extends Thread { private static final Log LOG = LogFactory.getLog(PolicyRefresher.class); + private static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("policy"); private final RangerBasePlugin plugIn; private final String serviceType; @@ -171,7 +172,13 @@ public class PolicyRefresher extends Thread { LOG.debug("==> PolicyRefresher(serviceName=" + serviceName + ").loadPolicy()"); } - //load policy from PolicyAmdin + RangerPerfTracer perf = null; + + if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "PolicyRefresher.loadPolicy(serviceName=" + serviceName + ")"); + } + + //load policy from PolicyAdmin ServicePolicies svcPolicies = loadPolicyfromPolicyAdmin(); if ( svcPolicies == null) { @@ -183,6 +190,8 @@ public class PolicyRefresher extends Thread { saveToCache(svcPolicies); } + RangerPerfTracer.log(perf); + if (svcPolicies != null) { plugIn.setPolicies(svcPolicies); policiesSetInPlugin = true; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java index b59ae1f..f199c44 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java @@ -219,7 +219,7 @@ public class TestPolicyEngine { policyEngineOptions.disableTagPolicyEvaluation = false; - policyEngine = new RangerPolicyEngineImpl("test-policyengine", servicePolicies, policyEngineOptions); + policyEngine = new RangerPolicyEngineImpl(testName, servicePolicies, policyEngineOptions); RangerAccessRequest request = null; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/agents-common/src/test/resources/log4j.properties b/agents-common/src/test/resources/log4j.properties deleted file mode 100644 index cb409e8..0000000 --- a/agents-common/src/test/resources/log4j.properties +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -##-- To prevent junits from cluttering the build run by default all test runs send output to null appender -log4j.appender.devnull=org.apache.log4j.varia.NullAppender -ranger.root.logger=FATAL,devnull - -##-- uncomment the following line during during development/debugging so see debug messages during test run to be emitted to console -# ranger.root.logger=DEBUG,console - -log4j.rootLogger=${ranger.root.logger} - -# Logging Threshold -log4j.threshold=ALL - -# -# console -# Add "console" to rootlogger above if you want to use this -# -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/agents-common/src/test/resources/log4j.xml ---------------------------------------------------------------------- diff --git a/agents-common/src/test/resources/log4j.xml b/agents-common/src/test/resources/log4j.xml new file mode 100644 index 0000000..48ed214 --- /dev/null +++ b/agents-common/src/test/resources/log4j.xml @@ -0,0 +1,53 @@ +<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> + +<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"; debug="false"> + + <appender name="console" class="org.apache.log4j.ConsoleAppender"> + <param name="target" value="System.err" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d{ISO8601} %-5p [%t] %c{2}: %m%n" /> + </layout> + </appender> + + <!-- + <appender name="perf_appender" class="org.apache.log4j.DailyRollingFileAppender"> + <param name="file" value="./ranger_admin_perf_test.log" /> + <param name="datePattern" value="'.'yyyy-MM-dd" /> + <param name="append" value="true" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d [%t] %m%n" /> + </layout> + </appender> + + <category name="ranger.perf" additivity="false"> + <priority value="info" /> + <appender-ref ref="perf_appender" /> + </category> + + <category name="ranger.perf.policy" additivity="false"> + <priority value="warn" /> + <appender-ref ref="perf_appender" /> + </category> + + <category name="ranger.perf.item" additivity="false"> + <priority value="warn" /> + <appender-ref ref="perf_appender" /> + </category> + + <category name="ranger.perf.condition" additivity="false"> + <priority value="warn" /> + <appender-ref ref="perf_appender" /> + </category> + + <category name="ranger.perf.enricher" additivity="false"> + <priority value="warn" /> + <appender-ref ref="perf_appender" /> + </category> + --> + + <root> + <priority value="warn" /> + <appender-ref ref="console" /> + </root> + +</log4j:configuration> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/security-admin/.gitignore ---------------------------------------------------------------------- diff --git a/security-admin/.gitignore b/security-admin/.gitignore index bf7dc37..5a3a673 100644 --- a/security-admin/.gitignore +++ b/security-admin/.gitignore @@ -1,8 +1,6 @@ /target/ /bin/ -/bin/ /target .settings/ .pydevproject -log4j.xml *.log http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java index e3adc1d..6e164f3 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java @@ -114,7 +114,7 @@ public class RangerPolicyRetriever { RangerPerfTracer perf = null; if(RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerPolicyRetriever.getServicePolicies(serviceName=" + serviceName + ", serviceId=" + serviceId + ")"); + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerPolicyRetriever.getServicePolicies(serviceName=" + serviceName + ",serviceId=" + serviceId + ")"); } if(xService != null) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java new file mode 100644 index 0000000..6cc4e5e --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java @@ -0,0 +1,597 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.biz; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.utils.StringUtil; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.entity.*; +import org.apache.ranger.plugin.model.*; +import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.util.RangerPerfTracer; + + +public class RangerTagDBRetriever { + static final Log LOG = LogFactory.getLog(RangerTagDBRetriever.class); + static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerTagDBRetriever"); + + private final RangerDaoManager daoMgr; + private final XXService xService; + private final LookupCache lookupCache; + + private List<RangerServiceResource> serviceResources; + private Map<Long, RangerTagDef> tagDefs; + private Map<Long, RangerTag> tags; + private List<RangerTagResourceMap> tagResourceMaps; + + + public RangerTagDBRetriever(final RangerDaoManager daoMgr, final XXService xService) { + this.daoMgr = daoMgr; + this.xService = xService; + this.lookupCache = new LookupCache(); + + + if (this.daoMgr != null && this.xService != null) { + + RangerPerfTracer perf = null; + + if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "RangerTagDBReceiver-Optimized(serviceName=" + xService.getName()); + } + + TagRetrieverServiceResourceContext serviceResourceContext = new TagRetrieverServiceResourceContext(xService); + TagRetrieverTagDefContext tagDefContext = new TagRetrieverTagDefContext(xService); + TagRetrieverTagContext tagContext = new TagRetrieverTagContext(xService); + + serviceResources = serviceResourceContext.getAllServiceResources(); + tagDefs = tagDefContext.getAllTagDefs(); + tags = tagContext.getAllTags(); + tagResourceMaps = getAllTagResourceMaps(); + + RangerPerfTracer.log(perf); + + } + } + + public List<RangerTagResourceMap> getTagResourceMaps() { + return tagResourceMaps; + } + + public List<RangerServiceResource> getServiceResources() { + return serviceResources; + } + + public Map<Long, RangerTagDef> getTagDefs() { + return tagDefs; + } + + public Map<Long, RangerTag> getTags() { + return tags; + } + + private List<RangerTagResourceMap> getAllTagResourceMaps() { + + List<XXTagResourceMap> xTagResourceMaps = daoMgr.getXXTagResourceMap().findByServiceId(xService.getId()); + ListIterator<XXTagResourceMap> iterTagResourceMap = xTagResourceMaps.listIterator(); + + List<RangerTagResourceMap> ret = new ArrayList<RangerTagResourceMap>(); + + while (iterTagResourceMap.hasNext()) { + + XXTagResourceMap xTagResourceMap = iterTagResourceMap.next(); + + if (xTagResourceMap != null) { + + RangerTagResourceMap tagResourceMap = new RangerTagResourceMap(); + + tagResourceMap.setId(xTagResourceMap.getId()); + tagResourceMap.setGuid(xTagResourceMap.getGuid()); + tagResourceMap.setCreatedBy(lookupCache.getUserScreenName(xTagResourceMap.getAddedByUserId())); + tagResourceMap.setUpdatedBy(lookupCache.getUserScreenName(xTagResourceMap.getUpdatedByUserId())); + tagResourceMap.setCreateTime(xTagResourceMap.getCreateTime()); + tagResourceMap.setUpdateTime(xTagResourceMap.getUpdateTime()); + tagResourceMap.setResourceId(xTagResourceMap.getResourceId()); + tagResourceMap.setTagId(xTagResourceMap.getTagId()); + + ret.add(tagResourceMap); + } + } + return ret; + } + + static <T> List<T> asList(T obj) { + List<T> ret = new ArrayList<T>(); + + if (obj != null) { + ret.add(obj); + } + + return ret; + } + + private class LookupCache { + final Map<Long, String> userScreenNames = new HashMap<Long, String>(); + final Map<Long, String> resourceDefs = new HashMap<Long, String>(); + + String getUserScreenName(Long userId) { + String ret = null; + + if (userId != null) { + ret = userScreenNames.get(userId); + + if (ret == null) { + XXPortalUser user = daoMgr.getXXPortalUser().getById(userId); + + if (user != null) { + ret = user.getPublicScreenName(); + + if (StringUtil.isEmpty(ret)) { + ret = user.getFirstName(); + + if (StringUtil.isEmpty(ret)) { + ret = user.getLoginId(); + } else { + if (!StringUtil.isEmpty(user.getLastName())) { + ret += (" " + user.getLastName()); + } + } + } + + if (ret != null) { + userScreenNames.put(userId, ret); + } + } + } + } + + return ret; + } + + String getResourceName(Long resourceDefId) { + String ret = null; + + if (resourceDefId != null) { + ret = resourceDefs.get(resourceDefId); + + if (ret == null) { + XXResourceDef xResourceDef = daoMgr.getXXResourceDef().getById(resourceDefId); + + if (xResourceDef != null) { + ret = xResourceDef.getName(); + + resourceDefs.put(resourceDefId, ret); + } + } + } + + return ret; + } + } + + private class TagRetrieverServiceResourceContext { + + final XXService service; + final ListIterator<XXServiceResource> iterServiceResource; + final ListIterator<XXServiceResourceElement> iterServiceResourceElement; + final ListIterator<XXServiceResourceElementValue> iterServiceResourceElementValue; + + TagRetrieverServiceResourceContext(XXService xService) { + Long serviceId = xService == null ? null : xService.getId(); + + List<XXServiceResource> xServiceResources = daoMgr.getXXServiceResource().findTaggedResourcesInServiceId(serviceId); + List<XXServiceResourceElement> xServiceResourceElements = daoMgr.getXXServiceResourceElement().findByServiceId(serviceId); + List<XXServiceResourceElementValue> xServiceResourceElementValues = daoMgr.getXXServiceResourceElementValue().findByServiceId(serviceId); + + this.service = xService; + this.iterServiceResource = xServiceResources.listIterator(); + this.iterServiceResourceElement = xServiceResourceElements.listIterator(); + this.iterServiceResourceElementValue = xServiceResourceElementValues.listIterator(); + + } + + TagRetrieverServiceResourceContext(XXServiceResource xServiceResource, XXService xService) { + Long resourceId = xServiceResource == null ? null : xServiceResource.getId(); + + List<XXServiceResource> xServiceResources = asList(xServiceResource); + List<XXServiceResourceElement> xServiceResourceElements = daoMgr.getXXServiceResourceElement().findByResourceId(resourceId); + List<XXServiceResourceElementValue> xServiceResourceElementValues = daoMgr.getXXServiceResourceElementValue().findByResourceId(resourceId); + + this.service = xService; + this.iterServiceResource = xServiceResources.listIterator(); + this.iterServiceResourceElement = xServiceResourceElements.listIterator(); + this.iterServiceResourceElementValue = xServiceResourceElementValues.listIterator(); + + } + + List<RangerServiceResource> getAllServiceResources() { + List<RangerServiceResource> ret = new ArrayList<RangerServiceResource>(); + + while (iterServiceResource.hasNext()) { + RangerServiceResource serviceResource = getNextServiceResource(); + + if (serviceResource != null) { + ret.add(serviceResource); + } + } + + if (!hasProcessedAll()) { + LOG.warn("getAllServiceResources(): perhaps one or more serviceResources got updated during retrieval. Using fallback ... "); + + ret = getServiceResourcesBySecondary(); + } + + return ret; + } + + RangerServiceResource getNextServiceResource() { + RangerServiceResource ret = null; + + if (iterServiceResource.hasNext()) { + XXServiceResource xServiceResource = iterServiceResource.next(); + + if (xServiceResource != null) { + ret = new RangerServiceResource(); + + ret.setId(xServiceResource.getId()); + ret.setGuid(xServiceResource.getGuid()); + ret.setIsEnabled(xServiceResource.getIsEnabled()); + ret.setCreatedBy(lookupCache.getUserScreenName(xServiceResource.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xServiceResource.getUpdatedByUserId())); + ret.setCreateTime(xServiceResource.getCreateTime()); + ret.setUpdateTime(xServiceResource.getUpdateTime()); + ret.setVersion(xServiceResource.getVersion()); + ret.setResourceSignature(xServiceResource.getResourceSignature()); + + getServiceResourceElements(ret); + } + } + + return ret; + } + + void getServiceResourceElements(RangerServiceResource serviceResource) { + while (iterServiceResourceElement.hasNext()) { + XXServiceResourceElement xServiceResourceElement = iterServiceResourceElement.next(); + + if (xServiceResourceElement.getResourceId().equals(serviceResource.getId())) { + RangerPolicyResource resource = new RangerPolicyResource(); + + resource.setIsExcludes(xServiceResourceElement.getIsExcludes()); + resource.setIsRecursive(xServiceResourceElement.getIsRecursive()); + + while (iterServiceResourceElementValue.hasNext()) { + XXServiceResourceElementValue xServiceResourceElementValue = iterServiceResourceElementValue.next(); + + if (xServiceResourceElementValue.getResElementId().equals(xServiceResourceElement.getId())) { + resource.getValues().add(xServiceResourceElementValue.getValue()); + } else { + if (iterServiceResourceElementValue.hasPrevious()) { + iterServiceResourceElementValue.previous(); + } + break; + } + } + + serviceResource.getResourceElements().put(lookupCache.getResourceName(xServiceResourceElement.getResDefId()), resource); + } else if (xServiceResourceElement.getResourceId().compareTo(serviceResource.getId()) > 0) { + if (iterServiceResourceElement.hasPrevious()) { + iterServiceResourceElement.previous(); + } + break; + } + } + } + + boolean hasProcessedAll() { + boolean moreToProcess = iterServiceResource.hasNext() + || iterServiceResourceElement.hasNext() + || iterServiceResourceElementValue.hasNext(); + return !moreToProcess; + } + + List<RangerServiceResource> getServiceResourcesBySecondary() { + List<RangerServiceResource> ret = null; + + if (service != null) { + List<XXServiceResource> xServiceResources = daoMgr.getXXServiceResource().findByServiceId(service.getId()); + + if (CollectionUtils.isNotEmpty(xServiceResources)) { + ret = new ArrayList<RangerServiceResource>(xServiceResources.size()); + + for (XXServiceResource xServiceResource : xServiceResources) { + TagRetrieverServiceResourceContext ctx = new TagRetrieverServiceResourceContext(xServiceResource, service); + + RangerServiceResource serviceResource = ctx.getNextServiceResource(); + + if (serviceResource != null) { + ret.add(serviceResource); + } + } + } + } + return ret; + } + } + + private class TagRetrieverTagDefContext { + + final XXService service; + final ListIterator<XXTagDef> iterTagDef; + final ListIterator<XXTagAttributeDef> iterTagAttributeDef; + + + TagRetrieverTagDefContext(XXService xService) { + Long serviceId = xService == null ? null : xService.getId(); + + List<XXTagDef> xTagDefs = daoMgr.getXXTagDef().findByServiceId(serviceId); + List<XXTagAttributeDef> xTagAttributeDefs = daoMgr.getXXTagAttributeDef().findByServiceId(serviceId); + + this.service = xService; + this.iterTagDef = xTagDefs.listIterator(); + this.iterTagAttributeDef = xTagAttributeDefs.listIterator(); + } + + TagRetrieverTagDefContext(XXTagDef xTagDef, XXService xService) { + Long tagDefId = xTagDef == null ? null : xTagDef.getId(); + + List<XXTagDef> xTagDefs = asList(xTagDef); + List<XXTagAttributeDef> xTagAttributeDefs = daoMgr.getXXTagAttributeDef().findByTagDefId(tagDefId); + + this.service = xService; + this.iterTagDef = xTagDefs.listIterator(); + this.iterTagAttributeDef = xTagAttributeDefs.listIterator(); + } + + Map<Long, RangerTagDef> getAllTagDefs() { + Map<Long, RangerTagDef> ret = new HashMap<Long, RangerTagDef>(); + + while (iterTagDef.hasNext()) { + RangerTagDef tagDef = getNextTagDef(); + + if (tagDef != null) { + ret.put(tagDef.getId(), tagDef); + } + } + + if (!hasProcessedAllTagDefs()) { + LOG.warn("getAllTagDefs(): perhaps one or more tag-definitions got updated during retrieval. Using fallback ... "); + + ret = getTagDefsBySecondary(); + + } + + return ret; + } + + RangerTagDef getNextTagDef() { + RangerTagDef ret = null; + + if (iterTagDef.hasNext()) { + XXTagDef xTagDef = iterTagDef.next(); + + if (xTagDef != null) { + ret = new RangerTagDef(); + + ret.setId(xTagDef.getId()); + ret.setGuid(xTagDef.getGuid()); + ret.setIsEnabled(xTagDef.getIsEnabled()); + ret.setCreatedBy(lookupCache.getUserScreenName(xTagDef.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xTagDef.getUpdatedByUserId())); + ret.setCreateTime(xTagDef.getCreateTime()); + ret.setUpdateTime(xTagDef.getUpdateTime()); + ret.setVersion(xTagDef.getVersion()); + ret.setName(xTagDef.getName()); + ret.setSource(xTagDef.getSource()); + + getTagAttributeDefs(ret); + } + } + + return ret; + } + + void getTagAttributeDefs(RangerTagDef tagDef) { + while (iterTagAttributeDef.hasNext()) { + XXTagAttributeDef xTagAttributeDef = iterTagAttributeDef.next(); + + if (xTagAttributeDef.getTagDefId().equals(tagDef.getId())) { + RangerTagDef.RangerTagAttributeDef tagAttributeDef = new RangerTagDef.RangerTagAttributeDef(); + + tagAttributeDef.setName(xTagAttributeDef.getName()); + tagAttributeDef.setType(xTagAttributeDef.getType()); + + tagDef.getAttributeDefs().add(tagAttributeDef); + } else if (xTagAttributeDef.getTagDefId().compareTo(tagDef.getId()) > 0) { + if (iterTagAttributeDef.hasPrevious()) { + iterTagAttributeDef.previous(); + } + break; + } + } + } + + boolean hasProcessedAllTagDefs() { + boolean moreToProcess = iterTagAttributeDef.hasNext(); + return !moreToProcess; + } + + Map<Long, RangerTagDef> getTagDefsBySecondary() { + Map<Long, RangerTagDef> ret = null; + + if (service != null) { + List<XXTagDef> xTagDefs = daoMgr.getXXTagDef().findByServiceId(service.getId()); + + if (CollectionUtils.isNotEmpty(xTagDefs)) { + ret = new HashMap<Long, RangerTagDef>(xTagDefs.size()); + + for (XXTagDef xTagDef : xTagDefs) { + TagRetrieverTagDefContext ctx = new TagRetrieverTagDefContext(xTagDef, service); + + RangerTagDef tagDef = ctx.getNextTagDef(); + + if (tagDef != null) { + ret.put(tagDef.getId(), tagDef); + } + } + } + } + return ret; + } + } + + private class TagRetrieverTagContext { + + final XXService service; + final ListIterator<XXTag> iterTag; + final ListIterator<XXTagAttribute> iterTagAttribute; + + TagRetrieverTagContext(XXService xService) { + Long serviceId = xService == null ? null : xService.getId(); + + List<XXTag> xTags = daoMgr.getXXTag().findByServiceId(serviceId); + List<XXTagAttribute> xTagAttributes = daoMgr.getXXTagAttribute().findByServiceId(serviceId); + + this.service = xService; + this.iterTag = xTags.listIterator(); + this.iterTagAttribute = xTagAttributes.listIterator(); + + } + + TagRetrieverTagContext(XXTag xTag, XXService xService) { + Long tagId = xTag == null ? null : xTag.getId(); + + List<XXTag> xTags = asList(xTag); + List<XXTagAttribute> xTagAttributes = daoMgr.getXXTagAttribute().findByTagId(tagId); + + this.service = xService; + this.iterTag = xTags.listIterator(); + this.iterTagAttribute = xTagAttributes.listIterator(); + } + + + Map<Long, RangerTag> getAllTags() { + Map<Long, RangerTag> ret = new HashMap<Long, RangerTag>(); + + while (iterTag.hasNext()) { + RangerTag tag = getNextTag(); + + if (tag != null) { + ret.put(tag.getId(), tag); + } + } + + if (!hasProcessedAllTags()) { + LOG.warn("getAllTags(): perhaps one or more tags got updated during retrieval. Using fallback ... "); + + ret = getTagsBySecondary(); + } + + return ret; + } + + RangerTag getNextTag() { + RangerTag ret = null; + + if (iterTag.hasNext()) { + XXTag xTag = iterTag.next(); + + if (xTag != null) { + ret = new RangerTag(); + + ret.setId(xTag.getId()); + ret.setGuid(xTag.getGuid()); + ret.setCreatedBy(lookupCache.getUserScreenName(xTag.getAddedByUserId())); + ret.setUpdatedBy(lookupCache.getUserScreenName(xTag.getUpdatedByUserId())); + ret.setCreateTime(xTag.getCreateTime()); + ret.setUpdateTime(xTag.getUpdateTime()); + ret.setVersion(xTag.getVersion()); + + Map<Long, RangerTagDef> tagDefs = getTagDefs(); + if (tagDefs != null) { + RangerTagDef tagDef = tagDefs.get(xTag.getType()); + if (tagDef != null) { + ret.setType(tagDef.getName()); + } + } + + getTagAttributes(ret); + } + } + + return ret; + } + + void getTagAttributes(RangerTag tag) { + while (iterTagAttribute.hasNext()) { + XXTagAttribute xTagAttribute = iterTagAttribute.next(); + + if (xTagAttribute.getTagId().equals(tag.getId())) { + String attributeName = xTagAttribute.getName(); + String attributeValue = xTagAttribute.getValue(); + + + tag.getAttributes().put(attributeName, attributeValue); + } else if (xTagAttribute.getTagId().compareTo(tag.getId()) > 0) { + if (iterTagAttribute.hasPrevious()) { + iterTagAttribute.previous(); + } + break; + } + } + } + + boolean hasProcessedAllTags() { + boolean moreToProcess = iterTagAttribute.hasNext(); + return !moreToProcess; + } + + Map<Long, RangerTag> getTagsBySecondary() { + Map<Long, RangerTag> ret = null; + + if (service != null) { + List<XXTag> xTags = daoMgr.getXXTag().findByServiceId(service.getId()); + + if (CollectionUtils.isNotEmpty(xTags)) { + ret = new HashMap<Long, RangerTag>(xTags.size()); + + for (XXTag xTag : xTags) { + TagRetrieverTagContext ctx = new TagRetrieverTagContext(xTag, service); + + RangerTag tag = ctx.getNextTag(); + + if (tag != null) { + ret.put(tag.getId(), tag); + } + } + } + } + return ret; + } + } +} + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java index 300ba8d..f89a434 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java @@ -846,26 +846,14 @@ public class TagDBStore extends AbstractTagStore { throw new Exception("service-def does not exist. id=" + xxService.getType()); } - List<RangerTagDef> tagDefs = rangerTagDefService.getTagDefsByServiceId(xxService.getId()); - List<RangerTag> tags = rangerTagService.getTagsByServiceId(xxService.getId()); - List<RangerServiceResource> resources = rangerServiceResourceService.getTaggedResourcesInServiceId(xxService.getId()); - List<RangerTagResourceMap> tagResourceMaps = rangerTagResourceMapService.getTagResourceMapsByServiceId(xxService.getId()); + RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, xxService); - Map<Long, RangerTagDef> tagDefMap = new HashMap<Long, RangerTagDef>(); - Map<Long, RangerTag> tagMap = new HashMap<Long, RangerTag>(); - Map<Long, List<Long>> resourceToTagIds = new HashMap<Long, List<Long>>(); - - if(CollectionUtils.isNotEmpty(tagDefs)) { - for(RangerTagDef tagDef : tagDefs) { - tagDefMap.put(tagDef.getId(), tagDef); - } - } + Map<Long, RangerTagDef> tagDefMap = tagDBRetriever.getTagDefs(); + Map<Long, RangerTag> tagMap = tagDBRetriever.getTags(); + List<RangerServiceResource> resources = tagDBRetriever.getServiceResources(); + List<RangerTagResourceMap> tagResourceMaps = tagDBRetriever.getTagResourceMaps(); - if(CollectionUtils.isNotEmpty(tags)) { - for(RangerTag tag : tags) { - tagMap.put(tag.getId(), tag); - } - } + Map<Long, List<Long>> resourceToTagIds = new HashMap<Long, List<Long>>(); if(CollectionUtils.isNotEmpty(tagResourceMaps)) { Long resourceId = null; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementDao.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementDao.java index 8e2baab..56abeaf 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementDao.java @@ -46,4 +46,16 @@ public class XXServiceResourceElementDao extends BaseDao<XXServiceResourceElemen } } + public List<XXServiceResourceElement> findByServiceId(Long serviceId) { + if (serviceId == null) { + return new ArrayList<XXServiceResourceElement>(); + } + try { + return getEntityManager().createNamedQuery("XXServiceResourceElement.findByServiceId", tClass) + .setParameter("serviceId", serviceId) + .getResultList(); + } catch (NoResultException e) { + return new ArrayList<XXServiceResourceElement>(); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementValueDao.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementValueDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementValueDao.java index 04942a7..48cdbbb 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementValueDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceElementValueDao.java @@ -58,4 +58,29 @@ public class XXServiceResourceElementValueDao extends BaseDao<XXServiceResourceE } } + @SuppressWarnings("unchecked") + public List<XXServiceResourceElementValue> findByServiceId(Long serviceId) { + if (serviceId == null) { + return new ArrayList<XXServiceResourceElementValue>(); + } + try { + return getEntityManager().createNamedQuery("XXServiceResourceElementValue.findByServiceId") + .setParameter("serviceId", serviceId).getResultList(); + } catch (NoResultException e) { + return new ArrayList<XXServiceResourceElementValue>(); + } + } + + @SuppressWarnings("unchecked") + public List<XXServiceResourceElementValue> findByResourceId(Long resourceId) { + if (resourceId == null) { + return new ArrayList<XXServiceResourceElementValue>(); + } + try { + return getEntityManager().createNamedQuery("XXServiceResourceElementValue.findByResourceId") + .setParameter("resourceId", resourceId).getResultList(); + } catch (NoResultException e) { + return new ArrayList<XXServiceResourceElementValue>(); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDao.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDao.java index 5ba3b74..c993477 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDao.java @@ -45,4 +45,27 @@ public class XXTagAttributeDao extends BaseDao<XXTagAttribute> { } } + public List<XXTagAttribute> findByServiceId(Long serviceId) { + if (serviceId == null) { + return new ArrayList<XXTagAttribute>(); + } + try { + return getEntityManager().createNamedQuery("XXTagAttribute.findByServiceId", tClass) + .setParameter("serviceId", serviceId).getResultList(); + } catch (NoResultException e) { + return new ArrayList<XXTagAttribute>(); + } + } + + public List<XXTagAttribute> findByResourceId(Long resourceId) { + if (resourceId == null) { + return new ArrayList<XXTagAttribute>(); + } + try { + return getEntityManager().createNamedQuery("XXTagAttribute.findByResourceId", tClass) + .setParameter("resourceId", resourceId).getResultList(); + } catch (NoResultException e) { + return new ArrayList<XXTagAttribute>(); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDefDao.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDefDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDefDao.java index c8cb91d..56b5d1a 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDefDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXTagAttributeDefDao.java @@ -45,4 +45,27 @@ public class XXTagAttributeDefDao extends BaseDao<XXTagAttributeDef> { } } + public List<XXTagAttributeDef> findByServiceId(Long serviceId) { + if (serviceId == null) { + return new ArrayList<XXTagAttributeDef>(); + } + try { + return getEntityManager().createNamedQuery("XXTagAttributeDef.findByServiceId", tClass) + .setParameter("serviceId", serviceId).getResultList(); + } catch (NoResultException e) { + return new ArrayList<XXTagAttributeDef>(); + } + } + + public List<XXTagAttributeDef> findByResourceId(Long resourceId) { + if (resourceId == null) { + return new ArrayList<XXTagAttributeDef>(); + } + try { + return getEntityManager().createNamedQuery("XXTagAttributeDef.findByResourceId", tClass) + .setParameter("resourceId", resourceId).getResultList(); + } catch (NoResultException e) { + return new ArrayList<XXTagAttributeDef>(); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/7a80c8e3/security-admin/src/main/java/org/apache/ranger/db/XXTagDefDao.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTagDefDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXTagDefDao.java index 28ddfde..9a3ed59 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXTagDefDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXTagDefDao.java @@ -100,4 +100,17 @@ public class XXTagDefDao extends BaseDao<XXTagDef> { return; } } + + public List<XXTagDef> findByResourceId(Long resourceId) { + if (resourceId == null) { + return new ArrayList<XXTagDef>(); + } + + try { + return getEntityManager().createNamedQuery("XXTagDef.findByResourceId", tClass) + .setParameter("resourceId", resourceId).getResultList(); + } catch (NoResultException e) { + return new ArrayList<XXTagDef>(); + } + } }
