Repository: incubator-ranger Updated Branches: refs/heads/master 682424276 -> ff4e2e7b6
RANGER-1162: optimize retrieval of tags for resource, using trie lookup Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/ff4e2e7b Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/ff4e2e7b Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/ff4e2e7b Branch: refs/heads/master Commit: ff4e2e7b6847679388a13ddd8396e191e0106f18 Parents: 6824242 Author: Madhan Neethiraj <[email protected]> Authored: Fri Aug 26 01:31:05 2016 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Mon Aug 29 16:32:56 2016 -0700 ---------------------------------------------------------------------- .../RangerServiceResourceMatcher.java | 47 +++- .../contextenricher/RangerTagEnricher.java | 103 ++++++- .../policyengine/RangerPolicyEngineImpl.java | 6 +- .../policyengine/RangerPolicyRepository.java | 8 +- .../RangerAbstractPolicyEvaluator.java | 60 ++-- .../policyevaluator/RangerPolicyEvaluator.java | 9 +- .../RangerDefaultPolicyResourceMatcher.java | 5 + .../RangerPolicyResourceEvaluator.java | 38 +++ .../RangerPolicyResourceMatcher.java | 2 + .../ranger/plugin/util/RangerResourceTrie.java | 273 ++++++++++++------- .../ranger/plugin/util/ServiceDefUtil.java | 37 +++ .../plugin/contextenricher/TestTagEnricher.java | 156 +++++++++++ .../plugin/policyengine/TestPolicyEngine.java | 3 +- .../contextenricher/test_tagenricher_hive.json | 77 ++++++ 14 files changed, 678 insertions(+), 146 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java index 5c1ae64..cf7b8e7 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java @@ -19,24 +19,63 @@ package org.apache.ranger.plugin.contextenricher; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceResource; import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; +import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceEvaluator; +import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; +import org.apache.ranger.plugin.util.ServiceDefUtil; -public class RangerServiceResourceMatcher { - private final RangerServiceResource serviceResource; +import java.util.Map; + +public class RangerServiceResourceMatcher implements RangerPolicyResourceEvaluator { + private final RangerServiceResource serviceResource; private final RangerPolicyResourceMatcher policyResourceMatcher; + private final Integer leafResourceLevel; public RangerServiceResourceMatcher(final RangerServiceResource serviceResource, RangerPolicyResourceMatcher policyResourceMatcher) { - this.serviceResource = serviceResource; + this.serviceResource = serviceResource; this.policyResourceMatcher = policyResourceMatcher; + this.leafResourceLevel = ServiceDefUtil.getLeafResourceLevel(getServiceDef(), getPolicyResource()); } public RangerServiceResource getServiceResource() { return serviceResource; } + @Override + public long getId() { + return serviceResource != null ? serviceResource.getId() :-1; + } + + @Override public RangerPolicyResourceMatcher getPolicyResourceMatcher() { return policyResourceMatcher; } + @Override + public Map<String, RangerPolicy.RangerPolicyResource> getPolicyResource() { + return serviceResource != null ? serviceResource.getResourceElements() : null; + } + + @Override + public RangerResourceMatcher getResourceMatcher(String resourceName) { + return policyResourceMatcher != null ? policyResourceMatcher.getResourceMatcher(resourceName) : null; + } + + @Override + public Integer getLeafResourceLevel() { + return leafResourceLevel; + } + + @Override + public int compareTo(RangerPolicyResourceEvaluator other) { + return Long.compare(getId(), other.getId()); + } + public boolean isMatch(RangerAccessResource requestedResource) { - return this.policyResourceMatcher.isExactHeadMatch(requestedResource); + return policyResourceMatcher != null ? policyResourceMatcher.isExactHeadMatch(requestedResource) : false; + } + + RangerServiceDef getServiceDef() { + return policyResourceMatcher != null ? policyResourceMatcher.getServiceDef() : null; } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java index c397ca1..5b60a53 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java @@ -27,6 +27,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceResource; import org.apache.ranger.plugin.model.RangerTag; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; @@ -34,12 +35,16 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; import org.apache.ranger.plugin.util.RangerAccessRequestUtil; import org.apache.ranger.plugin.util.RangerPerfTracer; +import org.apache.ranger.plugin.util.RangerResourceTrie; import org.apache.ranger.plugin.util.ServiceTags; import java.io.*; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; public class RangerTagEnricher extends RangerAbstractContextEnricher { private static final Log LOG = LogFactory.getLog(RangerTagEnricher.class); @@ -47,16 +52,15 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { private static final Log PERF_CONTEXTENRICHER_INIT_LOG = RangerPerfTracer.getPerfLogger("contextenricher.init"); public static final String TAG_REFRESHER_POLLINGINTERVAL_OPTION = "tagRefresherPollingInterval"; + public static final String TAG_RETRIEVER_CLASSNAME_OPTION = "tagRetrieverClassName"; + public static final String TAG_DISABLE_TRIE_PREFILTER_OPTION = "disableTrieLookupPrefilter"; - public static final String TAG_RETRIEVER_CLASSNAME_OPTION = "tagRetrieverClassName"; - - private RangerTagRefresher tagRefresher = null; - - private RangerTagRetriever tagRetriever = null; - - ServiceTags serviceTags = null; - - List<RangerServiceResourceMatcher> serviceResourceMatchers; + private RangerTagRefresher tagRefresher = null; + private RangerTagRetriever tagRetriever = null; + private ServiceTags serviceTags = null; + private List<RangerServiceResourceMatcher> serviceResourceMatchers = null; + private Map<String, RangerResourceTrie> serviceResourceTrie = null; + private boolean disableTrieLookupPrefilter = false; @Override public void init() { @@ -70,6 +74,8 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { long pollingIntervalMs = getLongOption(TAG_REFRESHER_POLLINGINTERVAL_OPTION, 60 * 1000); + disableTrieLookupPrefilter = getBooleanOption(TAG_DISABLE_TRIE_PREFILTER_OPTION, false); + if (StringUtils.isNotBlank(tagRetrieverClassName)) { try { @@ -162,8 +168,19 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { } + Map<String, RangerResourceTrie> serviceResourceTrie = null; + + if(!disableTrieLookupPrefilter) { + serviceResourceTrie = new HashMap<String, RangerResourceTrie>(); + + for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) { + serviceResourceTrie.put(resourceDef.getName(), new RangerResourceTrie(resourceDef, resourceMatchers)); + } + } + this.serviceResourceMatchers = resourceMatchers; - this.serviceTags = serviceTags; + this.serviceResourceTrie = serviceResourceTrie; + this.serviceTags = serviceTags; } @Override @@ -191,7 +208,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { } List<RangerTag> ret = null; - final List<RangerServiceResourceMatcher> serviceResourceMatchers = this.serviceResourceMatchers; + final List<RangerServiceResourceMatcher> serviceResourceMatchers = getEvaluators(resource); if (CollectionUtils.isNotEmpty(serviceResourceMatchers)) { @@ -226,6 +243,70 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { return ret; } + private List<RangerServiceResourceMatcher> getEvaluators(RangerAccessResource resource) { + List<RangerServiceResourceMatcher> ret = null; + + if(serviceResourceTrie == null) { + ret = serviceResourceMatchers; + } else { + Set<String> resourceKeys = resource == null ? null : resource.getKeys(); + + if (CollectionUtils.isNotEmpty(resourceKeys)) { + boolean isRetModifiable = false; + + for (String resourceName : resourceKeys) { + RangerResourceTrie trie = serviceResourceTrie.get(resourceName); + + if (trie == null) { // if no trie exists for this resource level, ignore and continue to next level + continue; + } + + List<RangerServiceResourceMatcher> resourceEvaluators = trie.getEvaluatorsForResource(resource.getValue(resourceName)); + + if (CollectionUtils.isEmpty(resourceEvaluators)) { // no policies for this resource, bail out + ret = null; + } else if (ret == null) { // initialize ret with policies found for this resource + ret = resourceEvaluators; + } else { // remove policies from ret that are not in resourceEvaluators + if (isRetModifiable) { + ret.retainAll(resourceEvaluators); + } else { + final List<RangerServiceResourceMatcher> shorterList; + final List<RangerServiceResourceMatcher> longerList; + + if (ret.size() < resourceEvaluators.size()) { + shorterList = ret; + longerList = resourceEvaluators; + } else { + shorterList = resourceEvaluators; + longerList = ret; + } + + ret = new ArrayList<>(shorterList); + ret.retainAll(longerList); + isRetModifiable = true; + } + } + + if (CollectionUtils.isEmpty(ret)) { // if no policy exists, bail out and return empty list + ret = null; + break; + } + } + } + } + + if(ret == null) { + ret = Collections.emptyList(); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagEnricher.getEvaluators(" + resource.getAsString() + "): evaluatorCount=" + ret.size()); + } + + return ret; + } + static private List<RangerTag> getTagsForServiceResource(final ServiceTags serviceTags, final RangerServiceResource serviceResource) { List<RangerTag> ret = new ArrayList<RangerTag>(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index d0e8d62..6d3645f 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@ -706,12 +706,12 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { @Override public void reorderPolicyEvaluators() { if (LOG.isDebugEnabled()) { - LOG.debug("==> reorderPolicyEvaluators()"); + LOG.debug("==> reorderEvaluators()"); } RangerPerfTracer perf = null; if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_REBALANCE_LOG)) { - perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_REBALANCE_LOG, "RangerPolicyEngine.reorderPolicyEvaluators()"); + perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_REBALANCE_LOG, "RangerPolicyEngine.reorderEvaluators()"); } if (MapUtils.isNotEmpty(policyEvaluatorsMap)) { for (Map.Entry<Long, RangerPolicyEvaluator> entry : policyEvaluatorsMap.entrySet()) { @@ -735,7 +735,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { RangerPerfTracer.log(perf); if (LOG.isDebugEnabled()) { - LOG.debug("<== reorderPolicyEvaluators()"); + LOG.debug("<== reorderEvaluators()"); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index d06aecd..d0b3f09 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -253,7 +253,7 @@ class RangerPolicyRepository { continue; } - List<RangerPolicyEvaluator> resourceEvaluators = trie.getPoliciesForResource(resource.getValue(resourceName)); + List<RangerPolicyEvaluator> resourceEvaluators = trie.getEvaluatorsForResource(resource.getValue(resourceName)); if(CollectionUtils.isEmpty(resourceEvaluators)) { // no policies for this resource, bail out ret = null; @@ -729,7 +729,7 @@ class RangerPolicyRepository { void reorderPolicyEvaluators() { if (LOG.isDebugEnabled()) { - LOG.debug("==> reorderPolicyEvaluators()"); + LOG.debug("==> reorderEvaluators()"); } if(disableTrieLookupPrefilter) { @@ -743,7 +743,7 @@ class RangerPolicyRepository { } if (LOG.isDebugEnabled()) { - LOG.debug("<== reorderPolicyEvaluators()"); + LOG.debug("<== reorderEvaluators()"); } } @@ -753,7 +753,7 @@ class RangerPolicyRepository { RangerResourceTrie trie = entry.getValue(); if(trie != null) { - trie.reorderPolicyEvaluators(); + trie.reorderEvaluators(); } } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java index 8f45661..9b48dfe 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java @@ -28,14 +28,19 @@ import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceEvaluator; +import org.apache.ranger.plugin.util.ServiceDefUtil; + +import java.util.Map; public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvaluator { private static final Log LOG = LogFactory.getLog(RangerAbstractPolicyEvaluator.class); - private RangerPolicy policy = null; - private RangerServiceDef serviceDef = null; - private int evalOrder = 0; - protected long usageCount = 0; + private RangerPolicy policy = null; + private RangerServiceDef serviceDef = null; + private Integer leafResourceLevel = null; + private int evalOrder = 0; + protected long usageCount = 0; protected boolean usageCountMutable = true; @@ -45,8 +50,9 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu LOG.debug("==> RangerAbstractPolicyEvaluator.init(" + policy + ", " + serviceDef + ")"); } - this.policy = policy; - this.serviceDef = serviceDef; + this.policy = policy; + this.serviceDef = serviceDef; + this.leafResourceLevel = ServiceDefUtil.getLeafResourceLevel(serviceDef, getPolicyResource()); if(LOG.isDebugEnabled()) { LOG.debug("<== RangerAbstractPolicyEvaluator.init(" + policy + ", " + serviceDef + ")"); @@ -54,6 +60,16 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu } @Override + public long getId() { + return policy != null ? policy.getId() :-1; + } + + @Override + public Map<String, RangerPolicy.RangerPolicyResource> getPolicyResource() { + return policy !=null ? policy.getResources() : null; + } + + @Override public RangerPolicy getPolicy() { return policy; } @@ -63,6 +79,11 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu return serviceDef; } + @Override + public Integer getLeafResourceLevel() { + return leafResourceLevel; + } + public boolean hasAllow() { return policy != null && CollectionUtils.isNotEmpty(policy.getPolicyItems()); } @@ -90,21 +111,28 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu } @Override - public int compareTo(RangerPolicyEvaluator other) { + public int compareTo(RangerPolicyResourceEvaluator obj) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerAbstractPolicyEvaluator.compareTo()"); + LOG.debug("==> RangerAbstractPolicyEvaluator.compareTo()"); } int result; - if (hasDeny() && !other.hasDeny()) { - result = -1; - } else if (!hasDeny() && other.hasDeny()) { - result = 1; - } else { - result = Long.compare(other.getUsageCount(), this.usageCount); - if (result == 0) { - result = Integer.compare(this.evalOrder, other.getEvalOrder()); + + if(obj instanceof RangerPolicyEvaluator) { + RangerPolicyEvaluator other = (RangerPolicyEvaluator)obj; + + if (hasDeny() && !other.hasDeny()) { + result = -1; + } else if (!hasDeny() && other.hasDeny()) { + result = 1; + } else { + result = Long.compare(other.getUsageCount(), this.usageCount); + if (result == 0) { + result = Integer.compare(this.evalOrder, other.getEvalOrder()); + } } + } else { + result = Long.compare(getId(), obj.getId()); } if(LOG.isDebugEnabled()) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java index 23069cf..b60e06e 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java @@ -33,11 +33,10 @@ import org.apache.ranger.plugin.policyengine.RangerDataMaskResult; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; import org.apache.ranger.plugin.policyengine.RangerResourceAccessInfo; import org.apache.ranger.plugin.policyengine.RangerRowFilterResult; -import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; -import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; +import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceEvaluator; -public interface RangerPolicyEvaluator extends Comparable<RangerPolicyEvaluator> { +public interface RangerPolicyEvaluator extends RangerPolicyResourceEvaluator { String EVALUATOR_TYPE_AUTO = "auto"; String EVALUATOR_TYPE_OPTIMIZED = "optimized"; String EVALUATOR_TYPE_CACHED = "cached"; @@ -66,10 +65,6 @@ public interface RangerPolicyEvaluator extends Comparable<RangerPolicyEvaluator> boolean isAuditEnabled(); - RangerPolicyResourceMatcher getPolicyResourceMatcher(); - - RangerResourceMatcher getResourceMatcher(String resourceName); - void evaluate(RangerAccessRequest request, RangerAccessResult result); void evaluate(RangerAccessRequest request, RangerDataMaskResult result); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java index 78589cd..8bde807 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java @@ -180,6 +180,11 @@ public class RangerDefaultPolicyResourceMatcher implements RangerPolicyResourceM } @Override + public RangerServiceDef getServiceDef() { + return serviceDef; + } + + @Override public RangerResourceMatcher getResourceMatcher(String resourceName) { return matchers != null ? matchers.get(resourceName) : null; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java new file mode 100644 index 0000000..eed58e1 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.policyresourcematcher; + + +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; + +import java.util.Map; + +public interface RangerPolicyResourceEvaluator extends Comparable<RangerPolicyResourceEvaluator> { + long getId(); + + RangerPolicyResourceMatcher getPolicyResourceMatcher(); + + Map<String, RangerPolicy.RangerPolicyResource> getPolicyResource(); + + RangerResourceMatcher getResourceMatcher(String resourceName); + + Integer getLeafResourceLevel(); +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java index 80da868..a8810e5 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java @@ -36,6 +36,8 @@ public interface RangerPolicyResourceMatcher { void init(); + RangerServiceDef getServiceDef(); + RangerResourceMatcher getResourceMatcher(String resourceName); boolean isMatch(RangerAccessResource resource); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java index a5ffd1a..2079487 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java @@ -23,10 +23,9 @@ package org.apache.ranger.plugin.util; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.model.RangerServiceDef; -import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; +import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceEvaluator; import org.apache.ranger.plugin.resourcematcher.RangerAbstractResourceMatcher; import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; @@ -37,7 +36,7 @@ import java.util.List; import java.util.Map; -public class RangerResourceTrie { +public class RangerResourceTrie<T extends RangerPolicyResourceEvaluator> { private static final Log LOG = LogFactory.getLog(RangerResourceTrie.class); private static final String DEFAULT_WILDCARD_CHARS = "*?"; @@ -48,7 +47,7 @@ public class RangerResourceTrie { private final String wildcardChars; private final TrieNode root; - public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List<RangerPolicyEvaluator> evaluators) { + public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef, List<T> evaluators) { if(LOG.isDebugEnabled()) { LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ")"); } @@ -63,31 +62,38 @@ public class RangerResourceTrie { this.wildcardChars = optWildcard ? DEFAULT_WILDCARD_CHARS : ""; this.root = new TrieNode(Character.valueOf((char)0)); - for(RangerPolicyEvaluator evaluator : evaluators) { - RangerPolicy policy = evaluator.getPolicy(); - Map<String, RangerPolicyResource> policyResources = policy != null ? policy.getResources() : null; + for(T evaluator : evaluators) { + Map<String, RangerPolicyResource> policyResources = evaluator.getPolicyResource(); RangerPolicyResource policyResource = policyResources != null ? policyResources.get(resourceName) : null; if(policyResource == null) { + if(evaluator.getLeafResourceLevel() != null && resourceDef.getLevel() != null && evaluator.getLeafResourceLevel() < resourceDef.getLevel()) { + root.addWildcardEvaluator(evaluator); + } + continue; } if(policyResource.getIsExcludes()) { - root.addWildcardPolicy(evaluator); + root.addWildcardEvaluator(evaluator); } else { RangerResourceMatcher resourceMatcher = evaluator.getResourceMatcher(resourceName); if(resourceMatcher != null && resourceMatcher.isMatchAny()) { - root.addWildcardPolicy(evaluator); + root.addWildcardEvaluator(evaluator); } else { - for (String resource : policyResource.getValues()) { - insert(resource, policyResource.getIsRecursive(), evaluator); + if(CollectionUtils.isNotEmpty(policyResource.getValues())) { + for (String resource : policyResource.getValues()) { + insert(resource, policyResource.getIsRecursive(), evaluator); + } } } } } - root.postSetup(); + root.postSetup(null); + + LOG.info(toString()); if(LOG.isDebugEnabled()) { LOG.debug("<== RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + "): " + toString()); @@ -98,12 +104,12 @@ public class RangerResourceTrie { return resourceName; } - public List<RangerPolicyEvaluator> getPoliciesForResource(String resource) { + public List<T> getEvaluatorsForResource(String resource) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerResourceTrie.getPoliciesForResource(" + resource + ")"); + LOG.debug("==> RangerResourceTrie.getEvaluatorsForResource(" + resource + ")"); } - List<RangerPolicyEvaluator> ret = null; + List<T> ret = null; TrieNode curr = root; @@ -113,8 +119,8 @@ public class RangerResourceTrie { TrieNode child = curr.getChild(ch); if(child == null) { - ret = curr.getWildcardPolicies(); - curr = null; // so that curr.getPolicies() will not be called below + ret = curr.getWildcardEvaluators(); + curr = null; // so that curr.getEvaluators() will not be called below break; } @@ -123,41 +129,46 @@ public class RangerResourceTrie { if(ret == null) { if(curr != null) { - ret = curr.getPolicies(); + ret = curr.getEvaluators(); } } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerResourceTrie.getPoliciesForResource(" + resource + "): evaluatorCount=" + (ret == null ? 0 : ret.size())); + LOG.debug("<== RangerResourceTrie.getEvaluatorsForResource(" + resource + "): evaluatorCount=" + (ret == null ? 0 : ret.size())); } return ret; } - public int getNodeCount() { - return root.getNodeCount(); + public TrieData getTrieData() { + TrieData ret = new TrieData(); + + root.populateTrieData(ret); + ret.maxDepth = getMaxDepth(); + + return ret; } public int getMaxDepth() { return root.getMaxDepth(); } - public void reorderPolicyEvaluators() { - root.reorderPolicyEvaluators(); + public void reorderEvaluators() { + root.reorderEvaluators(null); } - private Character getLookupChar(char ch) { - return optIgnoreCase ? Character.valueOf(Character.toLowerCase(ch)) : Character.valueOf(ch); + private final Character getLookupChar(char ch) { + if(optIgnoreCase) { + ch = Character.toLowerCase(ch); + } + + return Character.valueOf(ch); } - private void insert(String resource, boolean isRecursive, RangerPolicyEvaluator evaluator) { + private void insert(String resource, boolean isRecursive, T evaluator) { TrieNode curr = root; boolean isWildcard = false; - if(optIgnoreCase) { - resource = resource.toLowerCase(); - } - final int len = resource.length(); for(int i = 0; i < len; i++) { Character ch = getLookupChar(resource.charAt(i)); @@ -173,9 +184,9 @@ public class RangerResourceTrie { } if(isWildcard || isRecursive) { - curr.addWildcardPolicy(evaluator); + curr.addWildcardEvaluator(evaluator); } else { - curr.addPolicy(evaluator); + curr.addEvaluator(evaluator); } } @@ -183,20 +194,42 @@ public class RangerResourceTrie { public String toString() { StringBuilder sb = new StringBuilder(); - sb.append("nodeCount=").append(getNodeCount()); - sb.append("; maxDepth=").append(getMaxDepth()); - sb.append(Character.LINE_SEPARATOR); - root.toString("", sb); + TrieData trieData = getTrieData(); + + sb.append("resourceName=").append(resourceName); + sb.append("; optIgnoreCase=").append(optIgnoreCase); + sb.append("; optWildcard=").append(optWildcard); + sb.append("; wildcardChars=").append(wildcardChars); + sb.append("; nodeCount=").append(trieData.nodeCount); + sb.append("; leafNodeCount=").append(trieData.leafNodeCount); + sb.append("; singleChildNodeCount=").append(trieData.singleChildNodeCount); + sb.append("; maxDepth=").append(trieData.maxDepth); + sb.append("; evaluatorListCount=").append(trieData.evaluatorListCount); + sb.append("; wildcardEvaluatorListCount=").append(trieData.wildcardEvaluatorListCount); + sb.append("; evaluatorListRefCount=").append(trieData.evaluatorListRefCount); + sb.append("; wildcardEvaluatorListRefCount=").append(trieData.wildcardEvaluatorListRefCount); return sb.toString(); } + + public class TrieData { + int nodeCount = 0; + int leafNodeCount = 0; + int singleChildNodeCount = 0; + int maxDepth = 0; + int evaluatorListCount = 0; + int wildcardEvaluatorListCount = 0; + int evaluatorListRefCount = 0; + int wildcardEvaluatorListRefCount = 0; + } } -class TrieNode { - private final Character c; - private Map<Character, TrieNode> children = null; - private List<RangerPolicyEvaluator> policies = null; - private List<RangerPolicyEvaluator> wildcardPolicies = null; +class TrieNode<T extends RangerPolicyResourceEvaluator> { + private final Character c; + private Map<Character, TrieNode> children = null; + private List<T> evaluators = null; + private List<T> wildcardEvaluators = null; + private boolean isSharingParentWildcardEvaluators = false; TrieNode(Character c) { this.c = c; @@ -210,12 +243,12 @@ class TrieNode { return children; } - List<RangerPolicyEvaluator> getPolicies() { - return policies; + List<T> getEvaluators() { + return evaluators; } - List<RangerPolicyEvaluator> getWildcardPolicies() { - return wildcardPolicies; + List<T> getWildcardEvaluators() { + return wildcardEvaluators; } TrieNode getChild(Character c) { @@ -224,18 +257,38 @@ class TrieNode { return ret; } - int getNodeCount() { - int ret = 1; + void populateTrieData(RangerResourceTrie.TrieData trieData) { + trieData.nodeCount++; + + if(wildcardEvaluators != null) { + if(isSharingParentWildcardEvaluators) { + trieData.wildcardEvaluatorListRefCount++; + } else { + trieData.wildcardEvaluatorListCount++; + } + } + + if(evaluators != null) { + if(evaluators == wildcardEvaluators) { + trieData.evaluatorListRefCount++; + } else { + trieData.evaluatorListCount++; + } + } + + if(children != null && children.size() > 0) { + if(children.size() == 1) { + trieData.singleChildNodeCount++; + } - if(children != null) { for(Map.Entry<Character, TrieNode> entry : children.entrySet()) { TrieNode child = entry.getValue(); - ret += child.getNodeCount(); + child.populateTrieData(trieData); } + } else { + trieData.leafNodeCount++; } - - return ret; } int getMaxDepth() { @@ -271,69 +324,91 @@ class TrieNode { return child; } - void addPolicy(RangerPolicyEvaluator evaluator) { - if(policies == null) { - policies = new ArrayList<RangerPolicyEvaluator>(); + void addEvaluator(T evaluator) { + if(evaluators == null) { + evaluators = new ArrayList<T>(); } - if(!policies.contains(evaluator)) { - policies.add(evaluator); + if(!evaluators.contains(evaluator)) { + evaluators.add(evaluator); } } - void addPolicies(List<RangerPolicyEvaluator> evaluators) { - if(CollectionUtils.isNotEmpty(evaluators)) { - for(RangerPolicyEvaluator evaluator : evaluators) { - addPolicy(evaluator); - } + void addWildcardEvaluator(T evaluator) { + if(wildcardEvaluators == null) { + wildcardEvaluators = new ArrayList<T>(); + } + + if(!wildcardEvaluators.contains(evaluator)) { + wildcardEvaluators.add(evaluator); } } - void addWildcardPolicy(RangerPolicyEvaluator evaluator) { - if(wildcardPolicies == null) { - wildcardPolicies = new ArrayList<RangerPolicyEvaluator>(); + void postSetup(List<T> parentWildcardEvaluators) { + // finalize wildcard-evaluators list by including parent's wildcard evaluators + if(parentWildcardEvaluators != null) { + if(CollectionUtils.isEmpty(this.wildcardEvaluators)) { + this.wildcardEvaluators = parentWildcardEvaluators; + } else { + for (T evaluator : parentWildcardEvaluators) { + addWildcardEvaluator(evaluator); + } + } } + this.isSharingParentWildcardEvaluators = wildcardEvaluators == parentWildcardEvaluators; - if(!wildcardPolicies.contains(evaluator)) { - wildcardPolicies.add(evaluator); + // finalize evaluators list by including wildcard evaluators + if(wildcardEvaluators != null) { + if(CollectionUtils.isEmpty(this.evaluators)) { + this.evaluators = wildcardEvaluators; + } else { + for (T evaluator : wildcardEvaluators) { + addEvaluator(evaluator); + } + } } - } - void addWildcardPolicies(List<RangerPolicyEvaluator> evaluators) { - if(CollectionUtils.isNotEmpty(evaluators)) { - for(RangerPolicyEvaluator evaluator : evaluators) { - addWildcardPolicy(evaluator); + if(!isSharingParentWildcardEvaluators && CollectionUtils.isNotEmpty(wildcardEvaluators)) { + Collections.sort(wildcardEvaluators); + } + + if(evaluators != wildcardEvaluators && CollectionUtils.isNotEmpty(evaluators)) { + Collections.sort(evaluators); + } + + if(children != null) { + for(Map.Entry<Character, TrieNode> entry : children.entrySet()) { + TrieNode child = entry.getValue(); + + child.postSetup(wildcardEvaluators); } } } - void postSetup() { - addPolicies(wildcardPolicies); + void reorderEvaluators(List<T> parentWildcardEvaluators) { + boolean isEvaluatorsSameAsWildcardEvaluators = evaluators == wildcardEvaluators; - if(wildcardPolicies != null) { - Collections.sort(wildcardPolicies); + if(isSharingParentWildcardEvaluators) { + wildcardEvaluators = parentWildcardEvaluators; + } else { + wildcardEvaluators = getSortedCopy(wildcardEvaluators); } - if(policies != null) { - Collections.sort(policies); + if(isEvaluatorsSameAsWildcardEvaluators) { + evaluators = wildcardEvaluators; + } else { + evaluators = getSortedCopy(evaluators); } if(children != null) { for(Map.Entry<Character, TrieNode> entry : children.entrySet()) { TrieNode child = entry.getValue(); - child.addWildcardPolicies(wildcardPolicies); - - child.postSetup(); + child.reorderEvaluators(wildcardEvaluators); } } } - void reorderPolicyEvaluators() { - wildcardPolicies = getSortedCopy(wildcardPolicies); - policies = getSortedCopy(policies); - } - public void toString(String prefix, StringBuilder sb) { String nodeValue = prefix; @@ -343,18 +418,18 @@ class TrieNode { sb.append("nodeValue=").append(nodeValue); sb.append("; childCount=").append(children == null ? 0 : children.size()); - sb.append("; policies=[ "); - if(policies != null) { - for(RangerPolicyEvaluator evaluator : policies) { - sb.append(evaluator.getPolicy().getId()).append(" "); + sb.append("; evaluators=[ "); + if(evaluators != null) { + for(T evaluator : evaluators) { + sb.append(evaluator.getId()).append(" "); } } sb.append("]"); - sb.append("; wildcardPolicies=[ "); - if(wildcardPolicies != null) { - for(RangerPolicyEvaluator evaluator : wildcardPolicies) { - sb.append(evaluator.getPolicy().getId()).append(" "); + sb.append("; wildcardEvaluators=[ "); + if(wildcardEvaluators != null) { + for(T evaluator : wildcardEvaluators) { + sb.append(evaluator.getId()).append(" "); } } sb.append("]"); @@ -370,16 +445,16 @@ class TrieNode { } public void clear() { - children = null; - policies = null; - wildcardPolicies = null; + children = null; + evaluators = null; + wildcardEvaluators = null; } - private List<RangerPolicyEvaluator> getSortedCopy(List<RangerPolicyEvaluator> evaluators) { - final List<RangerPolicyEvaluator> ret; + private List<T> getSortedCopy(List<T> evaluators) { + final List<T> ret; if(CollectionUtils.isNotEmpty(evaluators)) { - ret = new ArrayList<RangerPolicyEvaluator>(wildcardPolicies); + ret = new ArrayList<T>(evaluators); Collections.sort(ret); } else { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java index eaf60b7..0a2b451 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java @@ -22,6 +22,7 @@ package org.apache.ranger.plugin.util; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; +import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerAccessTypeDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerDataMaskTypeDef; @@ -73,6 +74,42 @@ public class ServiceDefUtil { return serviceDef; } + public static RangerResourceDef getResourceDef(RangerServiceDef serviceDef, String resource) { + RangerResourceDef ret = null; + + if(serviceDef != null && resource != null && CollectionUtils.isNotEmpty(serviceDef.getResources())) { + for(RangerResourceDef resourceDef : serviceDef.getResources()) { + if(StringUtils.equalsIgnoreCase(resourceDef.getName(), resource)) { + ret = resourceDef; + break; + } + } + } + + return ret; + } + + public static Integer getLeafResourceLevel(RangerServiceDef serviceDef, Map<String, RangerPolicy.RangerPolicyResource> policyResource) { + Integer ret = null; + + if(serviceDef != null && policyResource != null) { + for(Map.Entry<String, RangerPolicy.RangerPolicyResource> entry : policyResource.entrySet()) { + String resource = entry.getKey(); + RangerResourceDef resourceDef = ServiceDefUtil.getResourceDef(serviceDef, resource); + + if(resourceDef != null && resourceDef.getLevel() != null) { + if(ret == null) { + ret = resourceDef.getLevel(); + } else if(ret < resourceDef.getLevel()) { + ret = resourceDef.getLevel(); + } + } + } + } + + return ret; + } + private static void normalizeDataMaskDef(RangerServiceDef serviceDef) { if(serviceDef != null && serviceDef.getDataMaskDef() != null) { List<RangerResourceDef> dataMaskResources = serviceDef.getDataMaskDef().getResources(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java b/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java new file mode 100644 index 0000000..30190ab --- /dev/null +++ b/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.contextenricher; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; +import org.apache.ranger.plugin.contextenricher.TestTagEnricher.TagEnricherTestCase.TestData; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.model.RangerServiceResource; +import org.apache.ranger.plugin.model.RangerTag; +import org.apache.ranger.plugin.model.RangerTagDef; +import org.apache.ranger.plugin.policyengine.*; +import org.apache.ranger.plugin.util.RangerAccessRequestUtil; +import org.apache.ranger.plugin.util.ServiceTags; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestTagEnricher { + static Gson gsonBuilder = null; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") + .setPrettyPrinting() + .registerTypeAdapter(RangerAccessResource.class, new RangerResourceDeserializer()) + .create(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Test + public void testTagEnricher_hive() { + String[] hiveTestResourceFiles = { "/contextenricher/test_tagenricher_hive.json" }; + + runTestsFromResourceFiles(hiveTestResourceFiles); + } + + private void runTestsFromResourceFiles(String[] resourceNames) { + for(String resourceName : resourceNames) { + InputStream inStream = this.getClass().getResourceAsStream(resourceName); + InputStreamReader reader = new InputStreamReader(inStream); + + runTests(reader, resourceName); + } + } + + private void runTests(InputStreamReader reader, String testName) { + TagEnricherTestCase testCase = gsonBuilder.fromJson(reader, TagEnricherTestCase.class); + + assertTrue("invalid input: " + testName, testCase != null && testCase.serviceDef != null && testCase.serviceResources != null && testCase.tests != null); + + ServiceTags serviceTags = new ServiceTags(); + serviceTags.setServiceName(testCase.serviceName); + serviceTags.setTagDefinitions(testCase.tagDefinitions); + serviceTags.setTags(testCase.tags); + serviceTags.setServiceResources(testCase.serviceResources); + serviceTags.setResourceToTagIds(testCase.resourceToTagIds); + + RangerTagEnricher tagEnricher = new RangerTagEnricher(); + + tagEnricher.setServiceName(testCase.serviceName); + tagEnricher.setServiceDef(testCase.serviceDef); + tagEnricher.setServiceTags(serviceTags); + + List<String> expectedTags = new ArrayList<String>(); + List<String> resultTags = new ArrayList<String>(); + + + for (TestData test : testCase.tests) { + RangerAccessRequestImpl request = new RangerAccessRequestImpl(test.resource, "", "testUser", null); + + tagEnricher.enrich(request); + + List<RangerTag> expected = test.result; + List<RangerTag> result = RangerAccessRequestUtil.getRequestTagsFromContext(request.getContext()); + + expectedTags.clear(); + if(expected != null) { + for (RangerTag tag : expected) { + expectedTags.add(tag.getType()); + } + Collections.sort(expectedTags); + } + + resultTags.clear(); + if(result != null) { + for(RangerTag tag : result) { + resultTags.add(tag.getType()); + } + Collections.sort(resultTags); + } + + assertEquals(test.name, expectedTags, resultTags); + } + } + + static class TagEnricherTestCase { + public String serviceName; + public RangerServiceDef serviceDef; + public Map<Long, RangerTagDef> tagDefinitions; + public Map<Long, RangerTag> tags; + public List<RangerServiceResource> serviceResources; + public Map<Long, List<Long>> resourceToTagIds; + public List<TestData> tests; + + + class TestData { + public String name; + public RangerAccessResource resource; + public List<RangerTag> result; + } + } + + static class RangerResourceDeserializer implements JsonDeserializer<RangerAccessResource> { + @Override + public RangerAccessResource deserialize(JsonElement jsonObj, Type type, + JsonDeserializationContext context) throws JsonParseException { + return gsonBuilder.fromJson(jsonObj, RangerAccessResourceImpl.class); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java index f825f40..9937757 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java @@ -58,7 +58,6 @@ import static org.junit.Assert.*; public class TestPolicyEngine { - static RangerPolicyEngine policyEngine = null; static Gson gsonBuilder = null; @BeforeClass @@ -304,7 +303,7 @@ public class TestPolicyEngine { trustedProxyAddresses[i] = trustedProxyAddresses[i].trim(); } } - policyEngine = new RangerPolicyEngineImpl(testName, servicePolicies, policyEngineOptions); + RangerPolicyEngine policyEngine = new RangerPolicyEngineImpl(testName, servicePolicies, policyEngineOptions); policyEngine.setUseForwardedIPAddress(useForwardedIPAddress); policyEngine.setTrustedProxyAddresses(trustedProxyAddresses); long requestCount = 0L; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/ff4e2e7b/agents-common/src/test/resources/contextenricher/test_tagenricher_hive.json ---------------------------------------------------------------------- diff --git a/agents-common/src/test/resources/contextenricher/test_tagenricher_hive.json b/agents-common/src/test/resources/contextenricher/test_tagenricher_hive.json new file mode 100644 index 0000000..317c651 --- /dev/null +++ b/agents-common/src/test/resources/contextenricher/test_tagenricher_hive.json @@ -0,0 +1,77 @@ +{ + "serviceName":"hivedev", + + "serviceDef":{ + "name":"hive", + "id":3, + "resources":[ + {"name":"database","level":1,"mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Database","description":"Hive Database"}, + {"name":"table","level":2,"parent":"database","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Table","description":"Hive Table"}, + {"name":"udf","level":2,"parent":"database","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive UDF","description":"Hive UDF"}, + {"name":"column","level":3,"parent":"table","mandatory":true,"lookupSupported":true,"matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher","matcherOptions":{"wildCard":true, "ignoreCase":true},"label":"Hive Column","description":"Hive Column"} + ], + "accessTypes":[ + {"name":"select","label":"Select"}, + {"name":"update","label":"Update"}, + {"name":"create","label":"Create"}, + {"name":"drop","label":"Drop"}, + {"name":"alter","label":"Alter"}, + {"name":"index","label":"Index"}, + {"name":"lock","label":"Lock"}, + {"name":"all","label":"All"} + ] + }, + + "tagDefinitions": { + "1":{"name":"PII"}, + "2":{"name":"EXPIRES_ON","attributeDefs":[{"name":"expiry_date","type":"date"}]}, + "3":{"name":"FINANCE"} + }, + + "tags": { + "1":{"type":"PII"}, + "2":{"type":"EXPIRES_ON","attributes":{"expiry_date":"2015/12/31"}}, + "3":{"type":"FINANCE"} + }, + + "serviceResources": [ + {"id":1,"serviceName":"cl1_hive","resourceElements":{"database":{"values":["hr"]},"table":{"values":["employee"]},"column":{"values":["ssn"]}}}, + {"id":2,"serviceName":"cl1_hive","resourceElements":{"database":{"values":["finance"]}}}, + {"id":3,"serviceName":"cl1_hive","resourceElements":{"database":{"values":["finance"]},"table":{"values":["tax_2010"]}}}, + {"id":4,"serviceName":"cl1_hive","resourceElements":{"database":{"values":["finance"]},"table":{"values":["tax_2010"]},"column":{"values":["ssn"]}}} + ], + + "resourceToTagIds": { + "1":[1], + "2":[3], + "3":[2], + "4":[1] + }, + + "tests":[ + {"name":"hr.employee.ssn", + "resource":{"elements":{"database":"hr","table":"employee","column":"ssn"}}, + "result":[{"type":"PII"}] + } + , + {"name":"hr.employee.id", + "resource":{"elements":{"database":"hr","table":"employee","column":"id"}}, + "result":[] + } + , + {"name":"finance.tax_2010", + "resource":{"elements":{"database":"finance","table":"tax_2010"}}, + "result":[{"type":"EXPIRES_ON"},{"type":"FINANCE"}] + } + , + {"name":"finance.tax_2010.ssn", + "resource":{"elements":{"database":"finance","table":"tax_2010","column":"ssn"}}, + "result":[{"type":"EXPIRES_ON"},{"type":"PII"},{"type":"FINANCE"}] + } + , + {"name":"finance.tax_2010.id", + "resource":{"elements":{"database":"finance","table":"tax_2010","column":"id"}}, + "result":[{"type":"EXPIRES_ON"},{"type":"FINANCE"}] + } + ] +}
