This is an automated email from the ASF dual-hosted git repository.
abhay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 74839e3 RANGER-2510: Support for Incremental tag updates to improve
performance - part 3
74839e3 is described below
commit 74839e3ed04125b312c09543953b100fb103a8bb
Author: Abhay Kulkarni <[email protected]>
AuthorDate: Wed Oct 16 17:45:10 2019 -0700
RANGER-2510: Support for Incremental tag updates to improve performance -
part 3
---
.../RangerServiceResourceMatcher.java | 13 ++----
.../plugin/contextenricher/RangerTagEnricher.java | 51 +++++++++++-----------
.../validation/RangerZoneResourceMatcher.java | 8 ++--
.../policyengine/RangerPolicyRepository.java | 12 +++--
.../RangerAbstractPolicyEvaluator.java | 8 ++--
.../RangerPolicyResourceEvaluator.java | 3 +-
.../ranger/plugin/util/RangerResourceTrie.java | 35 +++++++++------
.../apache/ranger/plugin/util/ServiceDefUtil.java | 18 ++++++++
8 files changed, 87 insertions(+), 61 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
index f9bbb12..7b02dd6 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerServiceResourceMatcher.java
@@ -37,12 +37,12 @@ public class RangerServiceResourceMatcher implements
RangerPolicyResourceEvaluat
private final RangerServiceResource serviceResource;
private final RangerPolicyResourceMatcher policyResourceMatcher;
- private final Integer leafResourceLevel;
+ private RangerServiceDef.RangerResourceDef leafResourceDef;
public RangerServiceResourceMatcher(final RangerServiceResource
serviceResource, RangerPolicyResourceMatcher policyResourceMatcher) {
this.serviceResource = serviceResource;
this.policyResourceMatcher = policyResourceMatcher;
- this.leafResourceLevel =
ServiceDefUtil.getLeafResourceLevel(getServiceDef(), getPolicyResource());
+ this.leafResourceDef =
ServiceDefUtil.getLeafResourceDef(policyResourceMatcher.getServiceDef(),
getPolicyResource());
}
public RangerServiceResource getServiceResource() { return
serviceResource; }
@@ -66,18 +66,13 @@ public class RangerServiceResourceMatcher implements
RangerPolicyResourceEvaluat
}
@Override
- public Integer getLeafResourceLevel() {
- return leafResourceLevel;
+ public boolean isAncestorOf(RangerServiceDef.RangerResourceDef
resourceDef) {
+ return
ServiceDefUtil.isAncestorOf(policyResourceMatcher.getServiceDef(),
leafResourceDef, resourceDef);
}
-
-
public RangerPolicyResourceMatcher.MatchType
getMatchType(RangerAccessResource requestedResource, Map<String, Object>
evalContext) {
return policyResourceMatcher != null ?
policyResourceMatcher.getMatchType(requestedResource, evalContext) :
RangerPolicyResourceMatcher.MatchType.NONE;
}
- RangerServiceDef getServiceDef() {
- return policyResourceMatcher != null ?
policyResourceMatcher.getServiceDef() : null;
- }
static class IdComparator implements
Comparator<RangerServiceResourceMatcher>, Serializable {
@Override
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
index 7434ec9..4e56f5c 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
@@ -467,41 +467,26 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
for (RangerServiceResource serviceResource :
changedServiceResources) {
- if (removeOldServiceResource(serviceResource,
resourceMatchers, serviceResourceTrie)) {
+ final boolean removedOldServiceResource =
MapUtils.isEmpty(serviceResource.getResourceElements()) ||
removeOldServiceResource(serviceResource, resourceMatchers,
serviceResourceTrie);
+ if (removedOldServiceResource) {
if
(!StringUtils.isEmpty(serviceResource.getResourceSignature())) {
RangerServiceResourceMatcher
resourceMatcher = createRangerServiceResourceMatcher(serviceResource,
serviceDefHelper, hierarchies);
if (resourceMatcher != null) {
- for (String resourceDefName :
serviceResource.getResourceElements().keySet()) {
-
-
RangerResourceTrie<RangerServiceResourceMatcher> trie =
serviceResourceTrie.get(resourceDefName);
-
- if (trie == null) {
-
List<RangerServiceDef.RangerResourceDef> resourceDefs =
serviceDef.getResources();
-
RangerServiceDef.RangerResourceDef found = null;
- for
(RangerServiceDef.RangerResourceDef resourceDef : resourceDefs) {
- if
(StringUtils.equals(resourceDef.getName(), resourceDefName)) {
-
found = resourceDef;
-
break;
- }
- }
- if (found !=
null) {
- trie =
new RangerResourceTrie<>(found, new ArrayList<>());
-
serviceResourceTrie.put(resourceDefName, trie);
- }
- }
+ for
(RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) {
+
+
RangerResourceTrie<RangerServiceResourceMatcher> trie =
serviceResourceTrie.get(resourceDef.getName());
if (trie != null) {
-
trie.add(serviceResource.getResourceElements().get(resourceDefName),
resourceMatcher);
+
trie.add(serviceResource.getResourceElements().get(resourceDef.getName()),
resourceMatcher);
if
(LOG.isDebugEnabled()) {
LOG.debug("Added resource-matcher for service-resource:[" + serviceResource +
"]");
}
} else {
-
LOG.error("Could not create resource-matcher for resource: [" + serviceResource
+ "]. Should NOT happen!!");
-
LOG.error("Setting tagVersion to -1 to ensure that in the next download all
tags are downloaded");
- isInError =
true;
+ trie = new
RangerResourceTrie<>(resourceDef, Collections.singletonList(resourceMatcher));
+
serviceResourceTrie.put(resourceDef.getName(), trie);
}
}
resourceMatchers.add(resourceMatcher);
@@ -526,6 +511,9 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
LOG.error("Error in processing tag-deltas. Will
continue to use old tags");
deltas.setTagVersion(-1L);
} else {
+ for (Map.Entry<String,
RangerResourceTrie<RangerServiceResourceMatcher>> entry :
serviceResourceTrie.entrySet()) {
+ entry.getValue().wrapUpUpdate();
+ }
enrichedServiceTags = new
EnrichedServiceTags(allServiceTags, resourceMatchers, serviceResourceTrie);
}
@@ -804,10 +792,13 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
private static Set<RangerTagForEval> getTagsForServiceResource(final
ServiceTags serviceTags, final RangerServiceResource serviceResource, final
RangerPolicyResourceMatcher.MatchType matchType) {
Set<RangerTagForEval> ret = new HashSet<>();
- final Long resourceId = serviceResource.getId();
-
+ final Long resourceId =
serviceResource.getId();
final Map<Long, List<Long>> resourceToTagIds =
serviceTags.getResourceToTagIds();
- final Map<Long, RangerTag> tags = serviceTags.getTags();
+ final Map<Long, RangerTag> tags =
serviceTags.getTags();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Looking for tags for resource-id:[" +
resourceId + "] in serviceTags:[" + serviceTags + "]");
+ }
if (resourceId != null && MapUtils.isNotEmpty(resourceToTagIds)
&& MapUtils.isNotEmpty(tags)) {
@@ -823,6 +814,14 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
ret.add(new
RangerTagForEval(tag, matchType));
}
}
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No tags mapping found for
resource:[" + resourceId + "]");
+ }
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("resourceId is null or
resourceToTagTds mapping is null or tags mapping is null!");
}
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
index c7f5bc4..2b570f6 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java
@@ -39,7 +39,7 @@ public class RangerZoneResourceMatcher implements
RangerPolicyResourceEvaluator
private final String
securityZoneName;
private final Map<String, RangerPolicy.RangerPolicyResource>
policyResource;
private final RangerPolicyResourceMatcher
policyResourceMatcher;
- private final Integer
leafResourceLevel;
+ private RangerServiceDef.RangerResourceDef
leafResourceDef;
public RangerZoneResourceMatcher(final String securityZoneName, final
Map<String, RangerPolicy.RangerPolicyResource> policyResource, final
RangerServiceDef serviceDef) {
@@ -77,7 +77,7 @@ public class RangerZoneResourceMatcher implements
RangerPolicyResourceEvaluator
this.securityZoneName = securityZoneName;
this.policyResourceMatcher = matcher;
this.policyResource = policyResource;
- this.leafResourceLevel =
ServiceDefUtil.getLeafResourceLevel(serviceDef, policyResource);
+ this.leafResourceDef = ServiceDefUtil.getLeafResourceDef(serviceDef,
getPolicyResource());
}
public String getSecurityZoneName() { return securityZoneName; }
@@ -101,8 +101,8 @@ public class RangerZoneResourceMatcher implements
RangerPolicyResourceEvaluator
}
@Override
- public Integer getLeafResourceLevel() {
- return leafResourceLevel;
+ public boolean isAncestorOf(RangerServiceDef.RangerResourceDef
resourceDef) {
+ return
ServiceDefUtil.isAncestorOf(policyResourceMatcher.getServiceDef(),
leafResourceDef, resourceDef);
}
@Override
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index abc57df..065120f 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -1118,6 +1118,14 @@ class RangerPolicyRepository {
RangerResourceTrie<RangerPolicyEvaluator> trie =
trieMap.get(resourceDefName);
+ if (trie == null) {
+ if (RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE ==
policyDeltaType || RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE ==
policyDeltaType) {
+ LOG.warn("policyDeltaType is not for POLICY_CREATE and
trie for resourceDef:[" + resourceDefName + "] was null! Should not have
happened!!");
+ }
+ trie = new RangerResourceTrie<>(resourceDef, new
ArrayList<>(), RangerPolicyEvaluator.EVAL_ORDER_COMPARATOR, true);
+ trieMap.put(resourceDefName, trie);
+ }
+
if (policyDeltaType ==
RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE) {
addEvaluatorToTrie(newEvaluator, trie, resourceDefName);
} else if (policyDeltaType ==
RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) {
@@ -1137,9 +1145,7 @@ class RangerPolicyRepository {
private void addEvaluatorToTrie(RangerPolicyEvaluator newEvaluator,
RangerResourceTrie<RangerPolicyEvaluator> trie, String resourceDefName) {
if (newEvaluator != null) {
RangerPolicy.RangerPolicyResource resource =
newEvaluator.getPolicyResource().get(resourceDefName);
- if (resource != null) {
- trie.add(resource, newEvaluator);
- }
+ trie.add(resource, newEvaluator);
}
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
index 4d6962f..689985c 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
@@ -37,7 +37,7 @@ public abstract class RangerAbstractPolicyEvaluator
implements RangerPolicyEvalu
private RangerPolicy policy;
private RangerServiceDef serviceDef;
- private Integer leafResourceLevel;
+ private RangerServiceDef.RangerResourceDef leafResourceDef;
private int evalOrder;
protected long usageCount;
protected boolean usageCountMutable = true;
@@ -51,7 +51,7 @@ public abstract class RangerAbstractPolicyEvaluator
implements RangerPolicyEvalu
this.policy = policy;
this.serviceDef = serviceDef;
- this.leafResourceLevel =
ServiceDefUtil.getLeafResourceLevel(serviceDef, getPolicyResource());
+ this.leafResourceDef =
ServiceDefUtil.getLeafResourceDef(serviceDef, getPolicyResource());
if(LOG.isDebugEnabled()) {
LOG.debug("<== RangerAbstractPolicyEvaluator.init(" +
policy + ", " + serviceDef + ")");
@@ -84,8 +84,8 @@ public abstract class RangerAbstractPolicyEvaluator
implements RangerPolicyEvalu
}
@Override
- public Integer getLeafResourceLevel() {
- return leafResourceLevel;
+ public boolean isAncestorOf(RangerServiceDef.RangerResourceDef
resourceDef) {
+ return ServiceDefUtil.isAncestorOf(serviceDef, leafResourceDef,
resourceDef);
}
public boolean hasAllow() {
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java
index 7d43b4b..9da9fac 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceEvaluator.java
@@ -21,6 +21,7 @@ package org.apache.ranger.plugin.policyresourcematcher;
import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher;
import java.util.Map;
@@ -34,5 +35,5 @@ public interface RangerPolicyResourceEvaluator {
RangerResourceMatcher getResourceMatcher(String resourceName);
- Integer getLeafResourceLevel();
+ boolean isAncestorOf(RangerServiceDef.RangerResourceDef resourceDef);
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
index 0cba882..1afd07d 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerResourceTrie.java
@@ -50,7 +50,7 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
private static final String DEFAULT_WILDCARD_CHARS = "*?";
private static final String TRIE_BUILDER_THREAD_COUNT =
"ranger.policyengine.trie.builder.thread.count";
- private final String resourceName;
+ private final RangerServiceDef.RangerResourceDef resourceDef;
private final boolean optIgnoreCase;
private final boolean optWildcard;
private final String wildcardChars;
@@ -99,7 +99,7 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
tokenReplaceSpecialChars += delimiterEscape;
}
- this.resourceName = resourceDef.getName();
+ this.resourceDef = resourceDef;
this.optIgnoreCase =
RangerAbstractResourceMatcher.getOptionIgnoreCase(matcherOptions);
this.optWildcard =
RangerAbstractResourceMatcher.getOptionWildCard(matcherOptions);
this.wildcardChars = optWildcard ? DEFAULT_WILDCARD_CHARS +
tokenReplaceSpecialChars : "" + tokenReplaceSpecialChars;
@@ -123,7 +123,7 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
if (TRACE_LOG.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
root.toString("", sb);
- TRACE_LOG.trace("Trie Dump from RangerResourceTrie.init(name=" +
resourceName + "):\n{" + sb.toString() + "}");
+ TRACE_LOG.trace("Trie Dump from RangerResourceTrie.init(name=" +
resourceDef.getName() + "):\n{" + sb.toString() + "}");
}
if(LOG.isDebugEnabled()) {
@@ -132,7 +132,7 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
}
public String getResourceName() {
- return resourceName;
+ return resourceDef.getName();
}
public List<T> getEvaluatorsForResource(Object resource) {
@@ -160,12 +160,18 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG,
"RangerResourceTrie.add(name=" + resource + ")");
}
- if (resource.getIsExcludes()) {
- root.addWildcardEvaluator(evaluator);
+ if (resource == null) {
+ if (evaluator.isAncestorOf(resourceDef)) {
+ root.addWildcardEvaluator(evaluator);
+ }
} else {
- if (CollectionUtils.isNotEmpty(resource.getValues())) {
- for (String value : resource.getValues()) {
- insert(root, value, resource.getIsRecursive(), evaluator);
+ if (resource.getIsExcludes()) {
+ root.addWildcardEvaluator(evaluator);
+ } else {
+ if (CollectionUtils.isNotEmpty(resource.getValues())) {
+ for (String value : resource.getValues()) {
+ insert(root, value, resource.getIsRecursive(),
evaluator);
+ }
}
}
}
@@ -372,10 +378,10 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
RangerPerfTracer perf = null;
if(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_INIT_LOG)) {
- perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG,
"RangerResourceTrie.copyTrie(name=" + other.resourceName + ")");
+ perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG,
"RangerResourceTrie.copyTrie(name=" + other.resourceDef.getName() + ")");
}
- this.resourceName = other.resourceName;
+ this.resourceDef = other.resourceDef;
this.optIgnoreCase = other.optIgnoreCase;
this.optWildcard = other.optWildcard;
this.wildcardChars = other.wildcardChars;
@@ -391,7 +397,7 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
if (TRACE_LOG.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
root.toString("", sb);
- TRACE_LOG.trace("Trie Dump from RangerResourceTrie.copyTrie(name="
+ other.resourceName + "):\n{" + sb.toString() + "}");
+ TRACE_LOG.trace("Trie Dump from RangerResourceTrie.copyTrie(name="
+ other.resourceDef.getName() + "):\n{" + sb.toString() + "}");
}
}
@@ -410,6 +416,7 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
final boolean isMultiThreaded =
builderThreadCount > 1;
final List<ResourceTrieBuilderThread> builderThreads;
final Map<Character, Integer> builderThreadMap;
+ final String resourceName =
resourceDef.getName();
int lastUsedThreadIndex = 0;
if (isMultiThreaded) {
@@ -431,7 +438,7 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
RangerPolicyResource policyResource = policyResources != null ?
policyResources.get(resourceName) : null;
if (policyResource == null) {
- if (evaluator.getLeafResourceLevel() != null &&
resourceDef.getLevel() != null && evaluator.getLeafResourceLevel() <
resourceDef.getLevel()) {
+ if (evaluator.isAncestorOf(resourceDef)) {
ret.addWildcardEvaluator(evaluator);
}
@@ -763,7 +770,7 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
TrieData trieData = getTrieData();
- sb.append("resourceName=").append(resourceName);
+ sb.append("resourceName=").append(resourceDef.getName());
sb.append("; optIgnoreCase=").append(optIgnoreCase);
sb.append("; optWildcard=").append(optWildcard);
sb.append("; wildcardChars=").append(wildcardChars);
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
index f383241..f82f65f 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceDefUtil.java
@@ -128,6 +128,24 @@ public class ServiceDefUtil {
return ret;
}
+ public static boolean isAncestorOf(RangerServiceDef serviceDef,
RangerResourceDef ancestor, RangerResourceDef descendant) {
+
+ boolean ret = false;
+
+ if (ancestor != null && descendant != null) {
+ final String ancestorName = ancestor.getName();
+
+ for (RangerResourceDef node = descendant; node != null; node =
ServiceDefUtil.getResourceDef(serviceDef, node.getParent())) {
+ if (StringUtils.equalsIgnoreCase(ancestorName,
node.getParent())) {
+ ret = true;
+ break;
+ }
+ }
+ }
+
+ return ret;
+ }
+
public static boolean isEmpty(RangerPolicy.RangerPolicyResource
policyResource) {
boolean ret = true;
if (policyResource != null) {
