This is an automated email from the ASF dual-hosted git repository.
abhay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 7aa065860 RANGER-4922: Reduce time to find tags associated with
multi-level resource
7aa065860 is described below
commit 7aa06586039a53c58d0d9b07572168b0fd6ae0fa
Author: Abhay Kulkarni <[email protected]>
AuthorDate: Thu Sep 5 14:34:31 2024 -0700
RANGER-4922: Reduce time to find tags associated with multi-level resource
---
.../RangerServiceResourceMatcher.java | 12 +-
.../plugin/contextenricher/RangerTagEnricher.java | 145 ++++++++++++++++++---
.../validation/RangerZoneResourceMatcher.java | 4 +
.../plugin/policyengine/RangerResourceTrie.java | 103 ++++++++++-----
.../gds/GdsSharedResourceEvaluator.java | 3 +
.../RangerAbstractPolicyEvaluator.java | 4 +
.../RangerResourceEvaluator.java | 2 +
.../util/RangerResourceEvaluatorsRetriever.java | 16 ++-
.../apache/ranger/plugin/util/ServiceDefUtil.java | 28 +++-
.../plugin/contextenricher/TestTagEnricher.java | 1 +
.../plugin/policyengine/TestPathResourceTrie.java | 3 +
distro/src/main/assembly/ranger-tools.xml | 4 +
ranger-tools/conf/logback-mem-sizing.xml | 21 +++
ranger-tools/pom.xml | 20 +++
.../org/apache/ranger/sizing/RangerMemSizing.java | 1 +
15 files changed, 312 insertions(+), 55 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
index e696db518..44cf3dcdf 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
@@ -19,8 +19,9 @@
package org.apache.ranger.plugin.contextenricher;
+import org.apache.commons.lang.StringUtils;
import org.apache.ranger.plugin.model.RangerPolicy;
-import org.apache.ranger.plugin.model.RangerServiceDef;
+import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef;
import org.apache.ranger.plugin.model.RangerServiceResource;
import
org.apache.ranger.plugin.policyengine.RangerAccessRequest.ResourceElementMatchingScope;
import org.apache.ranger.plugin.policyengine.RangerAccessResource;
@@ -38,12 +39,12 @@ public class RangerServiceResourceMatcher implements
RangerResourceEvaluator {
private final RangerServiceResource serviceResource;
private final RangerPolicyResourceMatcher policyResourceMatcher;
- private RangerServiceDef.RangerResourceDef leafResourceDef;
+ private final RangerResourceDef leafResourceDef;
public RangerServiceResourceMatcher(final RangerServiceResource
serviceResource, RangerPolicyResourceMatcher policyResourceMatcher) {
this.serviceResource = serviceResource;
this.policyResourceMatcher = policyResourceMatcher;
- this.leafResourceDef =
ServiceDefUtil.getLeafResourceDef(policyResourceMatcher.getServiceDef(),
getPolicyResource());
+ this.leafResourceDef =
ServiceDefUtil.getLeafResourceDef(policyResourceMatcher.getServiceDef(),
getPolicyResource(), true);
}
public RangerServiceResource getServiceResource() { return
serviceResource; }
@@ -67,7 +68,7 @@ public class RangerServiceResourceMatcher implements
RangerResourceEvaluator {
}
@Override
- public boolean isAncestorOf(RangerServiceDef.RangerResourceDef
resourceDef) {
+ public boolean isAncestorOf(RangerResourceDef resourceDef) {
return
ServiceDefUtil.isAncestorOf(policyResourceMatcher.getServiceDef(),
leafResourceDef, resourceDef);
}
@@ -86,4 +87,7 @@ public class RangerServiceResourceMatcher implements
RangerResourceEvaluator {
public String toString() {
return String.valueOf(getId());
}
+
+ @Override
+ public boolean isLeaf(String resourceName) { return
StringUtils.equals(resourceName, leafResourceDef.getName()); }
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
index a8fbc0215..92d2a7848 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
@@ -21,17 +21,28 @@ package org.apache.ranger.plugin.contextenricher;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
+import org.apache.commons.collections.Predicate;
import org.apache.commons.lang.StringUtils;
import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig;
import org.apache.ranger.authorization.utils.JsonUtils;
import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.model.RangerServiceDef;
+import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef;
import org.apache.ranger.plugin.model.RangerServiceResource;
import org.apache.ranger.plugin.model.RangerTag;
import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper;
-import org.apache.ranger.plugin.policyengine.*;
+import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
+import
org.apache.ranger.plugin.policyengine.RangerAccessRequest.ResourceElementMatchingScope;
+import
org.apache.ranger.plugin.policyengine.RangerAccessRequest.ResourceMatchingScope;
+import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl;
+import org.apache.ranger.plugin.policyengine.RangerAccessResource;
+import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;
+import org.apache.ranger.plugin.policyengine.RangerPluginContext;
+import org.apache.ranger.plugin.policyengine.RangerResourceTrie;
import
org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher;
import
org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher;
+import org.apache.ranger.plugin.policyresourcematcher.RangerResourceEvaluator;
import org.apache.ranger.plugin.util.DownloadTrigger;
import org.apache.ranger.plugin.util.DownloaderTask;
import org.apache.ranger.plugin.service.RangerAuthContext;
@@ -42,6 +53,7 @@ import org.apache.ranger.plugin.util.RangerReadWriteLock;
import org.apache.ranger.plugin.util.RangerResourceEvaluatorsRetriever;
import org.apache.ranger.plugin.util.RangerServiceNotFoundException;
import org.apache.ranger.plugin.util.RangerServiceTagsDeltaUtil;
+import org.apache.ranger.plugin.util.ServiceDefUtil;
import org.apache.ranger.plugin.util.ServiceTags;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -83,12 +95,13 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
private EnrichedServiceTags enrichedServiceTags;
private boolean
disableCacheIfServiceNotFound = true;
private boolean dedupStrings
= true;
+ private Timer tagDownloadTimer;
+ private RangerServiceDefHelper serviceDefHelper;
private final BlockingQueue<DownloadTrigger> tagDownloadQueue = new
LinkedBlockingQueue<>();
- private Timer tagDownloadTimer;
+ private final RangerReadWriteLock lock = new
RangerReadWriteLock(false);
+ private final CachedResourceEvaluators cache = new
CachedResourceEvaluators();
- private RangerServiceDefHelper serviceDefHelper;
- private RangerReadWriteLock lock = new
RangerReadWriteLock(false);
@Override
public void init() {
@@ -347,6 +360,8 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
}
setEnrichedServiceTagsInPlugin();
+ cache.clearCache();
+
RangerPerfTracer.logAlways(perf);
}
@@ -451,7 +466,7 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
if (!disableTrieLookupPrefilter) {
serviceResourceTrie = new HashMap<>();
- for (RangerServiceDef.RangerResourceDef
resourceDef : serviceDef.getResources()) {
+ for (RangerResourceDef resourceDef :
serviceDef.getResources()) {
serviceResourceTrie.put(resourceDef.getName(), new
RangerResourceTrie(resourceDef, resourceMatchers,
getPolicyEngineOptions().optimizeTagTrieForRetrieval,
getPolicyEngineOptions().optimizeTagTrieForSpace, null));
}
}
@@ -483,8 +498,8 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
RangerServiceResourceMatcher
resourceMatcher = createRangerServiceResourceMatcher(serviceResource,
serviceDefHelper, hierarchies, getPluginContext());
if (resourceMatcher != null) {
- for
(RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) {
-
RangerPolicy.RangerPolicyResource policyResource =
serviceResource.getResourceElements().get(resourceDef.getName());
+ for (RangerResourceDef
resourceDef : serviceDef.getResources()) {
+ RangerPolicyResource
policyResource =
serviceResource.getResourceElements().get(resourceDef.getName());
RangerResourceTrie<RangerServiceResourceMatcher> trie =
serviceResourceTrie.get(resourceDef.getName());
if
(LOG.isDebugEnabled()) {
@@ -543,7 +558,7 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
RangerAccessResourceImpl accessResource = new
RangerAccessResourceImpl();
- for (Map.Entry<String,
RangerPolicy.RangerPolicyResource> entry :
serviceResource.getResourceElements().entrySet()) {
+ for (Map.Entry<String, RangerPolicyResource> entry :
serviceResource.getResourceElements().entrySet()) {
accessResource.setValue(entry.getKey(),
entry.getValue().getValues());
}
@@ -583,7 +598,7 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
}
for (RangerServiceResourceMatcher matcher :
oldMatchers) {
- for (RangerServiceDef.RangerResourceDef
resourceDef : serviceDef.getResources()) {
+ for (RangerResourceDef resourceDef :
serviceDef.getResources()) {
String
resourceDefName = resourceDef.getName();
RangerResourceTrie<RangerServiceResourceMatcher> trie =
resourceTries.get(resourceDefName);
@@ -624,7 +639,7 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
if (isValidHierarchy == null) { // hierarchy not yet
validated
isValidHierarchy = Boolean.FALSE;
- for (List<RangerServiceDef.RangerResourceDef>
hierarchy : serviceDefHelper.getResourceHierarchies(policyType)) {
+ for (List<RangerResourceDef> hierarchy :
serviceDefHelper.getResourceHierarchies(policyType)) {
if
(serviceDefHelper.hierarchyHasAllResources(hierarchy, resourceKeys)) {
isValidHierarchy = Boolean.TRUE;
@@ -716,7 +731,7 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
if (request.isAccessTypeAny()) {
isMatched = matchType !=
RangerPolicyResourceMatcher.MatchType.NONE;
- } else if
(request.getResourceMatchingScope() ==
RangerAccessRequest.ResourceMatchingScope.SELF_OR_DESCENDANTS) {
+ } else if
(request.getResourceMatchingScope() ==
ResourceMatchingScope.SELF_OR_DESCENDANTS) {
isMatched = matchType !=
RangerPolicyResourceMatcher.MatchType.NONE;
} else {
isMatched = matchType ==
RangerPolicyResourceMatcher.MatchType.SELF || matchType ==
RangerPolicyResourceMatcher.MatchType.SELF_AND_ALL_DESCENDANTS || matchType ==
RangerPolicyResourceMatcher.MatchType.ANCESTOR;
@@ -752,14 +767,42 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
return ret;
}
+ private static class CachedResourceEvaluators {
+ private final Map<String, Map<Map<String,
ResourceElementMatchingScope>, Collection<RangerServiceResourceMatcher>>> cache
= new HashMap<>();
+ private final RangerReadWriteLock
cacheLock = new
RangerReadWriteLock(true);
+
+ CachedResourceEvaluators() {}
+
+ Collection<RangerServiceResourceMatcher> getEvaluators(String
resourceKey, Map<String, ResourceElementMatchingScope> scopes) {
+ Collection<RangerServiceResourceMatcher> ret;
+
+ try (RangerReadWriteLock.RangerLock ignored =
cacheLock.getReadLock()) {
+ ret = cache.getOrDefault(resourceKey,
Collections.emptyMap()).get(scopes);
+ }
+
+ return ret;
+ }
+
+ void cacheEvaluators(String resource, Map<String,
ResourceElementMatchingScope> scopes, Collection<RangerServiceResourceMatcher>
evaluators) {
+ try (RangerReadWriteLock.RangerLock ignored =
cacheLock.getWriteLock()) {
+ cache.computeIfAbsent(resource, k -> new
HashMap<>()).put(scopes, evaluators);
+ }
+ }
+
+ void clearCache() {
+ try (RangerReadWriteLock.RangerLock ignored =
cacheLock.getWriteLock()) {
+ cache.clear();
+ }
+ }
+ }
+
private Collection<RangerServiceResourceMatcher>
getEvaluators(RangerAccessRequest request, EnrichedServiceTags
enrichedServiceTags) {
if(LOG.isDebugEnabled()) {
LOG.debug("==>
RangerTagEnricher.getEvaluators(request=" + request + ")");
}
- Collection<RangerServiceResourceMatcher> ret;
-
- RangerAccessResource resource =
request.getResource();
+ Collection<RangerServiceResourceMatcher>
ret = null;
+ final RangerAccessResource
resource = request.getResource();
final Map<String,
RangerResourceTrie<RangerServiceResourceMatcher>> serviceResourceTrie =
enrichedServiceTags.getServiceResourceTrie();
if (resource == null || resource.getKeys() == null ||
resource.getKeys().isEmpty() || serviceResourceTrie == null) {
@@ -771,7 +814,27 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
perf =
RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG,
"RangerTagEnricher.getEvaluators(resource=" + resource.getAsString() + ")");
}
- ret =
RangerResourceEvaluatorsRetriever.getEvaluators(serviceResourceTrie,
resource.getAsMap(), request.getResourceElementMatchingScopes());
+ final Predicate predicate =
excludeDescendantMatches(request) ? new
SelfOrAncestorPredicate(serviceDefHelper.getResourceDef(resource.getLeafName()))
: null;
+
+ if (predicate != null) {
+ ret =
cache.getEvaluators(resource.getCacheKey(),
request.getResourceElementMatchingScopes());
+ }
+
+ if (ret == null) {
+ ret =
RangerResourceEvaluatorsRetriever.getEvaluators(serviceResourceTrie,
resource.getAsMap(), request.getResourceElementMatchingScopes(), predicate);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Found [" + ret.size() + "]
service-resource-matchers for service-resource [" + resource.getAsString() +
"]");
+ }
+
+ if (predicate != null) {
+
cache.cacheEvaluators(resource.getCacheKey(),
request.getResourceElementMatchingScopes(), ret);
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Found [" + ret.size() + "]
service-resource-matchers for service-resource [" + resource.getAsString() + "]
in the cache");
+ }
+ }
RangerPerfTracer.logAlways(perf);
}
@@ -786,6 +849,39 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
return ret;
}
+ private boolean excludeDescendantMatches(RangerAccessRequest request) {
+ final boolean ret;
+
+ if (request.isAccessTypeAny() ||
RangerAccessRequestUtil.getIsAnyAccessInContext(request.getContext())) {
+ ret = false;
+ } else {
+ RangerAccessResource resource = request.getResource();
+ String leafName = resource.getLeafName();
+
+ if (StringUtils.isNotEmpty(leafName)) {
+ RangerServiceDefHelper helper = new
RangerServiceDefHelper(getServiceDef());
+ Set<List<RangerResourceDef>> hierarchies =
helper.getResourceHierarchies(RangerPolicy.POLICY_TYPE_ACCESS,
resource.getKeys());
+
+ // skip caching if the leaf of accessed
resource is the deepest in the only applicable hierarchy
+ if (hierarchies.size() == 1) {
+ List<RangerResourceDef> theHierarchy
= hierarchies.iterator().next();
+ RangerResourceDef leafOfHierarchy
= theHierarchy.get(theHierarchy.size() - 1);
+
+ if
(StringUtils.equals(leafOfHierarchy.getName(), leafName)) {
+ ret = false;
+ } else {
+ ret = true;
+ }
+ } else {
+ ret = true;
+ }
+ } else {
+ ret = false;
+ }
+ }
+ return ret;
+ }
+
private static Set<RangerTagForEval> getTagsForServiceResource(Date
accessTime, final ServiceTags serviceTags, final RangerServiceResource
serviceResource, final RangerPolicyResourceMatcher.MatchType matchType) {
Set<RangerTagForEval> ret = new HashSet<>();
@@ -1141,4 +1237,23 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
}
}
}
+
+ private static class SelfOrAncestorPredicate implements Predicate {
+ private final RangerServiceDef.RangerResourceDef
leafResourceDef;
+
+ public
SelfOrAncestorPredicate(RangerServiceDef.RangerResourceDef leafResourceDef) {
+ this.leafResourceDef = leafResourceDef;
+ }
+
+ @Override
+ public boolean evaluate(Object o) {
+ if (o instanceof RangerResourceEvaluator) {
+ RangerResourceEvaluator evaluator =
(RangerResourceEvaluator) o;
+
+ return
evaluator.isLeaf(leafResourceDef.getName()) ||
evaluator.isAncestorOf(leafResourceDef);
+ }
+
+ return false;
+ }
+ }
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
index 1a8a867a0..27ed49b0b 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
@@ -19,6 +19,7 @@
package org.apache.ranger.plugin.model.validation;
+import org.apache.commons.lang.StringUtils;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.policyengine.RangerPluginContext;
@@ -109,6 +110,9 @@ public class RangerZoneResourceMatcher implements
RangerResourceEvaluator {
return
ServiceDefUtil.isAncestorOf(policyResourceMatcher.getServiceDef(),
leafResourceDef, resourceDef);
}
+ @Override
+ public boolean isLeaf(String resourceName) { return
StringUtils.equals(resourceName, leafResourceDef.getName()); }
+
@Override
public String toString() {
return "{security-zone-name:[" + securityZoneName + "],
policyResource=[" + policyResource +"]}";
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
index abfc8f0aa..3a3a80e53 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
@@ -21,6 +21,7 @@ package org.apache.ranger.plugin.policyengine;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.Predicate;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
@@ -36,15 +37,7 @@ import org.apache.ranger.plugin.util.ServiceDefUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.Stack;
+import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@@ -194,7 +187,11 @@ public class RangerResourceTrie<T extends
RangerResourceEvaluator> {
}
public Set<T> getEvaluatorsForResource(Object resource,
ResourceElementMatchingScope scope) {
- EvalCollector<T> ret = new EvalCollector<>();
+ return getEvaluatorsForResource(resource, scope, (Predicate) null);
+ }
+
+ public Set<T> getEvaluatorsForResource(Object resource,
ResourceElementMatchingScope scope, Predicate predicate) {
+ EvalCollector<T> ret = new EvalCollector<>(predicate);
traverse(resource, scope, ret);
@@ -202,7 +199,11 @@ public class RangerResourceTrie<T extends
RangerResourceEvaluator> {
}
public Set<T> getEvaluatorsForResource(Object resource,
ResourceElementMatchingScope scope, Set<T> filter) {
- EvalSubsetCollector<T> ret = new EvalSubsetCollector<>(filter);
+ return getEvaluatorsForResource(resource, scope, filter, null);
+ }
+
+ public Set<T> getEvaluatorsForResource(Object resource,
ResourceElementMatchingScope scope, Set<T> filter, Predicate predicate) {
+ EvalSubsetCollector<T> ret = new EvalSubsetCollector<>(filter,
predicate);
traverse(resource, scope, ret);
@@ -210,7 +211,11 @@ public class RangerResourceTrie<T extends
RangerResourceEvaluator> {
}
public int getEvaluatorsCountForResource(Object resource,
ResourceElementMatchingScope scope) {
- EvalCountCollector<T> ret = new EvalCountCollector<>();
+ return getEvaluatorsCountForResource(resource, scope, null);
+ }
+
+ public int getEvaluatorsCountForResource(Object resource,
ResourceElementMatchingScope scope, Predicate predicate) {
+ EvalCountCollector<T> ret = new EvalCountCollector<>(predicate);
traverse(resource, scope, ret);
@@ -1321,18 +1326,20 @@ public class RangerResourceTrie<T extends
RangerResourceEvaluator> {
}
}
- public interface TraverseMatchHandler<T> {
+ public interface TraverseMatchHandler<T extends RangerResourceEvaluator> {
// return: true - stop traverse, processing is complete
// false - continue traverse, processing is not complete yet
boolean process(Set<T> evaluators);
}
- public static class EvalCollector<T> implements TraverseMatchHandler<T> {
- private Set<T> result;
- private boolean isOwnedResult = false;
+ public static class EvalCollector<T extends RangerResourceEvaluator>
implements TraverseMatchHandler<T> {
+ private final Predicate predicate;
+ private Set<T> result;
+ private boolean isOwnedResult = false;
- public EvalCollector() {
- this.result = null;
+ public EvalCollector(Predicate predicate) {
+ this.predicate = predicate;
+ this.result = null;
}
public Set<T> getResult() {
@@ -1343,7 +1350,18 @@ public class RangerResourceTrie<T extends
RangerResourceEvaluator> {
public boolean process(Set<T> evaluators) {
if (evaluators != null && !evaluators.isEmpty()) {
if (result == null) {
- result = evaluators;
+ if (predicate == null) {
+ result = evaluators;
+ } else {
+ result = new HashSet<>();
+ isOwnedResult = true;
+
+ for (T evaluator : evaluators) {
+ if (predicate.evaluate(evaluator)) {
+ result.add(evaluator);
+ }
+ }
+ }
} else {
if (!isOwnedResult) {
result = new HashSet<>(result);
@@ -1351,7 +1369,15 @@ public class RangerResourceTrie<T extends
RangerResourceEvaluator> {
isOwnedResult = true;
}
- result.addAll(evaluators);
+ if (predicate == null) {
+ result.addAll(evaluators);
+ } else {
+ for (T evaluator : evaluators) {
+ if (predicate.evaluate(evaluator)) {
+ result.add(evaluator);
+ }
+ }
+ }
}
}
@@ -1359,13 +1385,15 @@ public class RangerResourceTrie<T extends
RangerResourceEvaluator> {
}
}
- public static class EvalSubsetCollector<T> implements
TraverseMatchHandler<T> {
- private final Set<T> filter;
- private Set<T> result;
+ public static class EvalSubsetCollector<T extends RangerResourceEvaluator>
implements TraverseMatchHandler<T> {
+ private final Predicate predicate;
+ private final Set<T> filter;
+ private Set<T> result;
- public EvalSubsetCollector(Set<T> filter) {
- this.filter = filter == null ? Collections.emptySet() : filter;
- this.result = null;
+ public EvalSubsetCollector(Set<T> filter, Predicate predicate) {
+ this.predicate = predicate;
+ this.filter = filter == null ? Collections.emptySet() : filter;
+ this.result = null;
}
public Set<T> getResult() {
@@ -1380,6 +1408,10 @@ public class RangerResourceTrie<T extends
RangerResourceEvaluator> {
}
intersect(filter, evaluators, result);
+
+ if (predicate != null) {
+ result.removeIf(evaluator ->
!predicate.evaluate(evaluator));
+ }
}
return result != null && (filter.size() == result.size()); // stop
traverse once the result includes all entries in the filter
@@ -1393,8 +1425,13 @@ public class RangerResourceTrie<T extends
RangerResourceEvaluator> {
}
}
- public static class EvalCountCollector<T> implements
TraverseMatchHandler<T> {
- private int result = 0;
+ public static class EvalCountCollector<T extends RangerResourceEvaluator>
implements TraverseMatchHandler<T> {
+ private final Predicate predicate;
+ private int result = 0;
+
+ public EvalCountCollector(Predicate predicate) {
+ this.predicate = predicate;
+ }
public int getResult() {
return result;
@@ -1403,7 +1440,15 @@ public class RangerResourceTrie<T extends
RangerResourceEvaluator> {
@Override
public boolean process(Set<T> evaluators) {
if (evaluators != null) {
- result += evaluators.size();
+ if (predicate == null) {
+ result += evaluators.size();
+ } else {
+ for (T evaluator : evaluators) {
+ if (predicate.evaluate(evaluator)) {
+ result++;
+ }
+ }
+ }
}
return false; // continue traverse
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsSharedResourceEvaluator.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsSharedResourceEvaluator.java
index c2773c9e0..309181344 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsSharedResourceEvaluator.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsSharedResourceEvaluator.java
@@ -105,6 +105,9 @@ public class GdsSharedResourceEvaluator implements
RangerResourceEvaluator {
return
ServiceDefUtil.isAncestorOf(policyResourceMatcher.getServiceDef(),
leafResourceDef, resourceDef);
}
+ @Override
+ public boolean isLeaf(String resourceName) { return
StringUtils.equals(leafResourceDef.getName(), resourceName); }
+
public Collection<String> getResourceKeys() {
return resource != null && resource.getResource() != null ?
resource.getResource().keySet() : Collections.emptySet();
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
index 549dc8f5a..006ceefc3 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
@@ -20,6 +20,7 @@
package org.apache.ranger.plugin.policyevaluator;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.ranger.plugin.model.RangerPolicy;
import
org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
@@ -609,5 +610,8 @@ public abstract class RangerAbstractPolicyEvaluator
implements RangerPolicyEvalu
return ServiceDefUtil.isAncestorOf(serviceDef,
leafResourceDef, resourceDef);
}
}
+
+ @Override
+ public boolean isLeaf(String resourceName) { return
StringUtils.equals(resourceName, leafResourceDef.getName()); }
}
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerResourceEvaluator.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerResourceEvaluator.java
index 014bdd528..7544f1bdd 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerResourceEvaluator.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerResourceEvaluator.java
@@ -36,4 +36,6 @@ public interface RangerResourceEvaluator {
RangerResourceMatcher getResourceMatcher(String resourceName);
boolean isAncestorOf(RangerServiceDef.RangerResourceDef resourceDef);
+
+ boolean isLeaf(String resourceName);
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceEvaluatorsRetriever.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceEvaluatorsRetriever.java
index 69cb4417d..7f39dc2a2 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceEvaluatorsRetriever.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceEvaluatorsRetriever.java
@@ -21,6 +21,7 @@ package org.apache.ranger.plugin.util;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
+import org.apache.commons.collections.Predicate;
import
org.apache.ranger.plugin.policyengine.RangerAccessRequest.ResourceElementMatchingScope;
import org.apache.ranger.plugin.policyengine.RangerResourceTrie;
import org.apache.ranger.plugin.policyresourcematcher.RangerResourceEvaluator;
@@ -40,6 +41,10 @@ public class RangerResourceEvaluatorsRetriever {
}
public static <T extends RangerResourceEvaluator> Collection<T>
getEvaluators(Map<String, RangerResourceTrie<T>> resourceTrie, Map<String, ?>
resource, Map<String, ResourceElementMatchingScope> scopes) {
+ return getEvaluators(resourceTrie, resource, scopes, null);
+ }
+
+ public static <T extends RangerResourceEvaluator> Collection<T>
getEvaluators(Map<String, RangerResourceTrie<T>> resourceTrie, Map<String, ?>
resource, Map<String, ResourceElementMatchingScope> scopes, Predicate
predicate) {
if (LOG.isDebugEnabled()) {
LOG.debug("==>
RangerPolicyResourceEvaluatorsRetriever.getEvaluators(" + resource + ")");
}
@@ -65,13 +70,18 @@ public class RangerResourceEvaluatorsRetriever {
Object resourceValues = resource.get(resourceDefName);
- int evalCount =
trie.getEvaluatorsCountForResource(resourceValues, scopes.get(resourceDefName));
+ int evalCount =
trie.getEvaluatorsCountForResource(resourceValues, scopes.get(resourceDefName),
predicate);
if (resourceWithMinEvals == null || (evalCount <
minEvalCount)) {
resourceWithMinEvals = resourceDefName;
minEvalCount = evalCount;
}
}
+
+ if (minEvalCount == 0) {
+ resourceWithMinEvals = null;
+ ret = Collections.emptySet();
+ }
} else if (resourceKeys.size() == 1) { // skip
getEvaluatorsCountForResource() when there is only one resource
String resourceKey =
resourceKeys.iterator().next();
RangerResourceTrie<T> trie =
resourceTrie.get(resourceKey);
@@ -84,7 +94,7 @@ public class RangerResourceEvaluatorsRetriever {
if (resourceWithMinEvals != null) {
RangerResourceTrie<T> trie =
resourceTrie.get(resourceWithMinEvals);
- ret =
trie.getEvaluatorsForResource(resource.get(resourceWithMinEvals),
scopes.get(resourceWithMinEvals));
+ ret =
trie.getEvaluatorsForResource(resource.get(resourceWithMinEvals),
scopes.get(resourceWithMinEvals), predicate);
for (String resourceDefName : resourceKeys) {
if (resourceWithMinEvals.equals(resourceDefName)) {
@@ -97,7 +107,7 @@ public class RangerResourceEvaluatorsRetriever {
continue;
}
- Set<T> evaluators =
trie.getEvaluatorsForResource(resource.get(resourceDefName),
scopes.get(resourceDefName), ret);
+ Set<T> evaluators =
trie.getEvaluatorsForResource(resource.get(resourceDefName),
scopes.get(resourceDefName), ret, predicate);
if (CollectionUtils.isEmpty(evaluators)) {
ret = Collections.emptySet();
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
index ed1ab5bda..2bb667e31 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
@@ -43,6 +43,7 @@ import
org.apache.ranger.plugin.model.RangerServiceDef.RangerDataMaskTypeDef;
import org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef;
import org.apache.ranger.plugin.policyengine.RangerPluginContext;
import org.apache.ranger.plugin.policyengine.RangerRequestScriptEvaluator;
+import org.apache.ranger.plugin.resourcematcher.RangerAbstractResourceMatcher;
import org.apache.ranger.plugin.store.AbstractServiceStore;
import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil;
import org.apache.ranger.plugin.util.ServicePolicies.SecurityZoneInfo;
@@ -188,6 +189,10 @@ public class ServiceDefUtil {
}
public static RangerResourceDef getLeafResourceDef(RangerServiceDef
serviceDef, Map<String, RangerPolicyResource> policyResource) {
+ return getLeafResourceDef(serviceDef, policyResource, false);
+ }
+
+ public static RangerResourceDef getLeafResourceDef(RangerServiceDef
serviceDef, Map<String, RangerPolicyResource> policyResource, boolean
excludeWildcardLeaves) {
RangerResourceDef ret = null;
if(serviceDef != null && policyResource != null) {
@@ -197,10 +202,12 @@ public class ServiceDefUtil {
RangerResourceDef resourceDef =
ServiceDefUtil.getResourceDef(serviceDef, resource);
if (resourceDef != null && resourceDef.getLevel() != null)
{
- if (ret == null) {
- ret = resourceDef;
- } else if(ret.getLevel() < resourceDef.getLevel()) {
- ret = resourceDef;
+ if (ret == null || ret.getLevel() <
resourceDef.getLevel()) {
+ if (StringUtils.isEmpty(resourceDef.getParent())) {
+ ret = resourceDef;
+ } else if (!excludeWildcardLeaves ||
!hasWildcardValue(entry.getValue().getValues())) {
+ ret = resourceDef;
+ }
}
}
}
@@ -210,6 +217,19 @@ public class ServiceDefUtil {
return ret;
}
+ private static boolean hasWildcardValue(List<String> values) {
+ boolean ret = false;
+
+ for (String strValue : values) {
+ ret = StringUtils.equals(strValue,
RangerAbstractResourceMatcher.WILDCARD_ASTERISK);
+ if (ret) {
+ break;
+ }
+ }
+
+ return ret;
+ }
+
public static boolean isAncestorOf(RangerServiceDef serviceDef,
RangerResourceDef ancestor, RangerResourceDef descendant) {
boolean ret = false;
diff --git
a/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java
b/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java
index 86985177a..3e0812a0f 100644
---
a/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java
+++
b/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java
@@ -128,6 +128,7 @@ public class TestTagEnricher {
for (TestData test : testCase.tests) {
RangerAccessRequestImpl request = new
RangerAccessRequestImpl(test.resource, test.accessType, "testUser", null, null);
+
((RangerMutableResource)request.getResource()).setServiceDef(testCase.serviceDef);
tagEnricher.enrich(request);
List<RangerTag> expected = test.result;
diff --git
a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPathResourceTrie.java
b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPathResourceTrie.java
index d23509f86..24e1a80cc 100644
---
a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPathResourceTrie.java
+++
b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPathResourceTrie.java
@@ -191,6 +191,9 @@ public class TestPathResourceTrie {
return false;
}
+ @Override
+ public boolean isLeaf(String resourceName) { return true; }
+
@Override
public String toString() {
return "id=" + id + ", resource=" + policyResource;
diff --git a/distro/src/main/assembly/ranger-tools.xml
b/distro/src/main/assembly/ranger-tools.xml
index 6893366d3..f12159ff9 100644
--- a/distro/src/main/assembly/ranger-tools.xml
+++ b/distro/src/main/assembly/ranger-tools.xml
@@ -72,6 +72,10 @@
<include>net.java.dev.jna:jna-platform:jar:${jna-platform.version}</include>
<include>org.slf4j:slf4j-api</include>
<include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include>
+
<include>org.slf4j:log4j-over-slf4j:jar:${slf4j.version}</include>
+
<include>ch.qos.logback:logback-classic:jar:${logback.version}</include>
+
<include>ch.qos.logback:logback-core:jar:${logback.version}</include>
+ <include>com.google.code.gson:gson:jar:${gson.version}</include>
</includes>
</dependencySet>
</dependencySets>
diff --git a/ranger-tools/conf/logback-mem-sizing.xml
b/ranger-tools/conf/logback-mem-sizing.xml
index c3b601451..700fcb538 100644
--- a/ranger-tools/conf/logback-mem-sizing.xml
+++ b/ranger-tools/conf/logback-mem-sizing.xml
@@ -39,6 +39,27 @@
<level>INFO</level>
</filter>
</appender>
+ <appender name="perf_appender"
class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <!--See http://logback.qos.ch/manual/appenders.html#RollingFileAppender-->
+ <!--and
http://logback.qos.ch/manual/appenders.html#TimeBasedRollingPolicy-->
+ <!--for further documentation-->
+ <file>./ranger-mem-sizing-perf.log</file>
+ <append>true</append>
+ <encoder>
+ <pattern>%d [%t] %msg%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+
<fileNamePattern>./ranger-mem-sizing-perf.log.%d{yyyy-MM-dd}</fileNamePattern>
+ <maxHistory>15</maxHistory>
+ <cleanHistoryOnStart>true</cleanHistoryOnStart>
+ </rollingPolicy>
+ </appender>
+ <logger name="org.apache.ranger.perf.tagenricher.tags.retrieval"
additivity="false" level="info">
+ <appender-ref ref="perf_appender"/>
+ </logger>
+ <logger name="org.apache.ranger.perf.policyresourcematcher.gethierarchy"
additivity="false" level="info">
+ <appender-ref ref="perf_appender"/>
+ </logger>
<root level="INFO">
<appender-ref ref="file"/>
</root>
diff --git a/ranger-tools/pom.xml b/ranger-tools/pom.xml
index 440f3711e..150924505 100644
--- a/ranger-tools/pom.xml
+++ b/ranger-tools/pom.xml
@@ -60,6 +60,26 @@
<artifactId>chart</artifactId>
<version>2.2.0</version>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>${logback.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ <version>${logback.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
+ </dependency>
<!-- Test -->
<dependency>
diff --git
a/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java
b/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java
index 4cb15c099..6272388e3 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java
@@ -619,6 +619,7 @@ public class RangerMemSizing {
ret.disablePolicyRefresher = true;
ret.disableTagRetriever = true;
ret.disableUserStoreRetriever = true;
+ ret.disableGdsInfoRetriever = true;
ret.optimizeTrieForSpace = optimizationMode.equals(OPT_MODE_SPACE);
ret.optimizeTrieForRetrieval = !ret.optimizeTrieForSpace;
ret.optimizeTagTrieForSpace = ret.optimizeTrieForSpace;
