This is an automated email from the ASF dual-hosted git repository.
abhay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 92890a1 RANGER-3107: Optimize memory requirements for storing
Tag/Policy/Zone trie - Trade-off processing time for memory
92890a1 is described below
commit 92890a199d2957e0849cab4b099847dc90958bba
Author: Abhay Kulkarni <[email protected]>
AuthorDate: Tue Dec 8 10:40:24 2020 -0800
RANGER-3107: Optimize memory requirements for storing Tag/Policy/Zone trie
- Trade-off processing time for memory
---
.../plugin/contextenricher/RangerTagEnricher.java | 95 +++++++++++--------
.../validation/RangerSecurityZoneValidator.java | 105 ++++++++++++---------
.../model/validation/RangerServiceDefHelper.java | 73 ++++++++++++++
.../ranger/plugin/policyengine/PolicyEngine.java | 93 +++++++++---------
.../policyengine/RangerPolicyRepository.java | 96 ++++++++++---------
.../plugin/policyengine/RangerResourceTrie.java | 45 +++++++--
.../plugin/contextenricher/TestTagEnricher.java | 1 +
.../policyengine/TestPolicyEngineComparison.java | 4 +
8 files changed, 324 insertions(+), 188 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
index d1afd6f..f31b2c5 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java
@@ -85,6 +85,8 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
private final BlockingQueue<DownloadTrigger> tagDownloadQueue = new
LinkedBlockingQueue<>();
private Timer tagDownloadTimer;
+ private RangerServiceDefHelper serviceDefHelper;
+
@Override
public void init() {
if (LOG.isDebugEnabled()) {
@@ -99,6 +101,8 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
disableTrieLookupPrefilter =
getBooleanOption(TAG_DISABLE_TRIE_PREFILTER_OPTION, false);
+ serviceDefHelper = new RangerServiceDefHelper(serviceDef,
false);
+
if (StringUtils.isNotBlank(tagRetrieverClassName)) {
try {
@@ -366,7 +370,6 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
enrichedServiceTags = null;
} else {
- RangerServiceDefHelper serviceDefHelper = new
RangerServiceDefHelper(serviceDef, false);
ResourceHierarchies hierarchies = new
ResourceHierarchies();
List<RangerServiceResourceMatcher> resourceMatchers =
new ArrayList<>();
@@ -408,7 +411,6 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
boolean isInError = false;
- RangerServiceDefHelper serviceDefHelper = new
RangerServiceDefHelper(serviceDef, false);
ResourceHierarchies hierarchies = new
ResourceHierarchies();
List<RangerServiceResourceMatcher>
resourceMatchers = new ArrayList<>();
@@ -506,8 +508,11 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
for (RangerServiceResourceMatcher matcher :
oldMatchers) {
- for (String resourceDefName :
serviceResource.getResourceElements().keySet()) {
+ for (RangerServiceDef.RangerResourceDef
resourceDef : serviceDef.getResources()) {
+ String resourceDefName =
resourceDef.getName();
+
RangerResourceTrie<RangerServiceResourceMatcher> trie =
resourceTries.get(resourceDefName);
+
if (trie != null) {
trie.delete(serviceResource.getResourceElements().get(resourceDefName),
matcher);
} else {
@@ -673,16 +678,13 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
if (resource == null || resource.getKeys() == null ||
resource.getKeys().isEmpty() || serviceResourceTrie == null) {
ret = enrichedServiceTags.getServiceResourceMatchers();
} else {
- Set<RangerServiceResourceMatcher> evaluators = null;
-
RangerPerfTracer perf = null;
if
(RangerPerfTracer.isPerfTraceEnabled(PERF_TRIE_OP_LOG)) {
perf =
RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG,
"RangerTagEnricher.getEvaluators(resource=" + resource.getAsString() + ")");
}
- Set<String> resourceKeys =
resource.getKeys();
- List<Set<RangerServiceResourceMatcher>>
serviceResourceMatchersList = null;
+ List<String> resourceKeys =
serviceDefHelper.getOrderedResourceNames(resource.getKeys());
Set<RangerServiceResourceMatcher> smallestList =
null;
if (CollectionUtils.isNotEmpty(resourceKeys)) {
@@ -694,47 +696,58 @@ public class RangerTagEnricher extends
RangerAbstractContextEnricher {
continue;
}
- Set<RangerServiceResourceMatcher>
serviceResourceMatchers =
trie.getEvaluatorsForResource(resource.getValue(resourceName));
-
- if
(CollectionUtils.isEmpty(serviceResourceMatchers)) { // no tags for this
resource, bail out
- serviceResourceMatchersList =
null;
- smallestList = null;
- break;
- }
-
- if (smallestList == null) {
- smallestList =
serviceResourceMatchers;
- } else {
- if (serviceResourceMatchersList
== null) {
-
serviceResourceMatchersList = new ArrayList<>();
-
serviceResourceMatchersList.add(smallestList);
+ Set<RangerServiceResourceMatcher>
serviceResourceMatchersForResource =
trie.getEvaluatorsForResource(resource.getValue(resourceName));
+ Set<RangerServiceResourceMatcher>
inheritedResourceMatchers = trie.getInheritedEvaluators();
+
+ if (smallestList != null) {
+ if
(CollectionUtils.isEmpty(inheritedResourceMatchers) &&
CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
+ smallestList = null;
+ } else if
(CollectionUtils.isEmpty(inheritedResourceMatchers)) {
+
smallestList.retainAll(serviceResourceMatchersForResource);
+ } else if
(CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
+
smallestList.retainAll(inheritedResourceMatchers);
+ } else {
+
Set<RangerServiceResourceMatcher> smaller, bigger;
+ if
(serviceResourceMatchersForResource.size() < inheritedResourceMatchers.size()) {
+ smaller =
serviceResourceMatchersForResource;
+ bigger =
inheritedResourceMatchers;
+ } else {
+ smaller =
inheritedResourceMatchers;
+ bigger =
serviceResourceMatchersForResource;
+ }
+
Set<RangerServiceResourceMatcher> tmp = new HashSet<>();
+ if (smallestList.size()
< smaller.size()) {
+
smallestList.stream().filter(smaller::contains).forEach(tmp::add);
+
smallestList.stream().filter(bigger::contains).forEach(tmp::add);
+ } else {
+
smaller.stream().filter(smallestList::contains).forEach(tmp::add);
+ if
(smallestList.size() < bigger.size()) {
+
smallestList.stream().filter(bigger::contains).forEach(tmp::add);
+ } else {
+
bigger.stream().filter(smallestList::contains).forEach(tmp::add);
+ }
+ }
+ smallestList = tmp;
}
-
serviceResourceMatchersList.add(serviceResourceMatchers);
-
- if (smallestList.size() >
serviceResourceMatchers.size()) {
- smallestList =
serviceResourceMatchers;
+ } else {
+ if
(CollectionUtils.isEmpty(inheritedResourceMatchers) ||
CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
+
Set<RangerServiceResourceMatcher> tmp =
CollectionUtils.isEmpty(inheritedResourceMatchers) ?
serviceResourceMatchersForResource : inheritedResourceMatchers;
+ smallestList =
resourceKeys.size() == 1 || CollectionUtils.isEmpty(tmp) ? tmp : new
HashSet<>(tmp);
+ } else {
+ smallestList = new
HashSet<>(serviceResourceMatchersForResource);
+
smallestList.addAll(inheritedResourceMatchers);
}
}
- }
- if (serviceResourceMatchersList != null) {
- evaluators = new
HashSet<>(smallestList);
- for (Set<RangerServiceResourceMatcher>
serviceResourceMatchers : serviceResourceMatchersList) {
- if (serviceResourceMatchers !=
smallestList) {
- // remove other
serviceResourceMatchers from ret that are not in serviceResourceMatchers
-
evaluators.retainAll(serviceResourceMatchers);
- if
(CollectionUtils.isEmpty(evaluators)) { // if no policy exists, bail out and
return empty list
- evaluators =
null;
- break;
- }
- }
+
+ if
(CollectionUtils.isEmpty(smallestList)) {// no tags for this resource, bail out
+ smallestList = null;
+ break;
}
- } else {
- evaluators = smallestList;
}
}
- if (evaluators != null) {
- ret = new ArrayList<>(evaluators);
+ if (smallestList != null) {
+ ret = new ArrayList<>(smallestList);
}
RangerPerfTracer.logAlways(perf);
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerSecurityZoneValidator.java
b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerSecurityZoneValidator.java
index 2db2f22..0c2dcee 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerSecurityZoneValidator.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerSecurityZoneValidator.java
@@ -247,17 +247,17 @@ public class RangerSecurityZoneValidator extends
RangerValidator {
}
if (securityZone.getServices() != null) {
- for (Map.Entry<String, RangerSecurityZoneService>
serviceResouceMapEntry : securityZone.getServices()
+ for (Map.Entry<String, RangerSecurityZoneService>
serviceResourceMapEntry : securityZone.getServices()
.entrySet()) {
- if
(serviceResouceMapEntry.getValue().getResources() != null) {
- for (Map<String, List<String>> resource
: serviceResouceMapEntry.getValue().getResources()) {
+ if
(serviceResourceMapEntry.getValue().getResources() != null) {
+ for (Map<String, List<String>> resource
: serviceResourceMapEntry.getValue().getResources()) {
if (resource != null) {
for (Map.Entry<String,
List<String>> entry : resource.entrySet()) {
if
(CollectionUtils.isEmpty(entry.getValue())) {
ValidationErrorCode error =
ValidationErrorCode.SECURITY_ZONE_VALIDATION_ERR_MISSING_RESOURCES;
failures.add(new ValidationFailureDetailsBuilder().field("security zone
resources")
.subField("resources").isMissing()
-
.becauseOf(error.getMessage(serviceResouceMapEntry.getKey()))
+
.becauseOf(error.getMessage(serviceResourceMapEntry.getKey()))
.errorCode(error.getErrorCode()).build());
ret =
false;
}
@@ -406,70 +406,81 @@ public class RangerSecurityZoneValidator extends
RangerValidator {
// check each evaluator to see if the resource-match actually
happens. If yes then add the zone-evaluator to matching evaluators.
// flag error if there are more than one matching evaluators
with different zone-ids.
//
+
+ RangerServiceDefHelper serviceDefHelper = new
RangerServiceDefHelper(serviceDef, true);
+
for (RangerSecurityZone zone : zones) {
List<HashMap<String, List<String>>> resources =
zone.getServices().get(serviceName).getResources();
for (Map<String, List<String>> resource : resources) {
- List<Set<RangerZoneResourceMatcher>> zoneMatchersList = null;
Set<RangerZoneResourceMatcher> smallestList = null;
- for (Map.Entry<String, List<String>> entry :
resource.entrySet()) {
- String resourceDefName = entry.getKey();
- List<String> resourceValues = entry.getValue();
+ List<String> resourceKeys =
serviceDefHelper.getOrderedResourceNames(resource.keySet());
- RangerResourceTrie<RangerZoneResourceMatcher> trie
= trieMap.get(resourceDefName);
- Set<RangerZoneResourceMatcher> matchedZones
= trie.getEvaluatorsForResource(resourceValues);
+ for (String resourceDefName : resourceKeys) {
+ List<String> resourceValues =
resource.get(resourceDefName);
- if (LOG.isDebugEnabled()) {
- LOG.debug("ResourceDefName:[" + resourceDefName +"],
values:[" + resourceValues +"], matched-zones:[" + matchedZones +"]");
- }
- if (CollectionUtils.isEmpty(matchedZones)) { // no
policies for this resource, bail out
- zoneMatchersList = null;
- smallestList = null;
- break;
- }
+ RangerResourceTrie<RangerZoneResourceMatcher> trie =
trieMap.get(resourceDefName);
- if (smallestList == null) {
- smallestList = matchedZones;
- } else {
- if (zoneMatchersList == null) {
- zoneMatchersList = new ArrayList<>();
- zoneMatchersList.add(smallestList);
- }
- zoneMatchersList.add(matchedZones);
+ Set<RangerZoneResourceMatcher> zoneMatchersForResource =
trie.getEvaluatorsForResource(resourceValues);
+ Set<RangerZoneResourceMatcher> inheritedZoneMatchers =
trie.getInheritedEvaluators();
- if (smallestList.size() > matchedZones.size()) {
- smallestList = matchedZones;
- }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ResourceDefName:[" + resourceDefName + "],
values:[" + resourceValues + "], matched-zones:[" + zoneMatchersForResource +
"], inherited-zones:[" + inheritedZoneMatchers + "]");
}
- }
- if (smallestList == null) {
- continue;
- }
- final Set<RangerZoneResourceMatcher> intersection;
-
- if (zoneMatchersList != null) {
- intersection = new HashSet<>(smallestList);
- for (Set<RangerZoneResourceMatcher> zoneMatchers :
zoneMatchersList) {
- if (zoneMatchers != smallestList) {
- // remove zones from intersection that are not in
zoneMatchers
- intersection.retainAll(zoneMatchers);
- if (CollectionUtils.isEmpty(intersection)) { // if
no zoneMatcher exists, bail out and return empty list
- break;
+
+ if (smallestList != null) {
+ if (CollectionUtils.isEmpty(inheritedZoneMatchers) &&
CollectionUtils.isEmpty(zoneMatchersForResource)) {
+ smallestList = null;
+ } else if
(CollectionUtils.isEmpty(inheritedZoneMatchers)) {
+ smallestList.retainAll(zoneMatchersForResource);
+ } else if
(CollectionUtils.isEmpty(zoneMatchersForResource)) {
+ smallestList.retainAll(inheritedZoneMatchers);
+ } else {
+ Set<RangerZoneResourceMatcher> smaller, bigger;
+ if (zoneMatchersForResource.size() <
inheritedZoneMatchers.size()) {
+ smaller = zoneMatchersForResource;
+ bigger = inheritedZoneMatchers;
+ } else {
+ smaller = inheritedZoneMatchers;
+ bigger = zoneMatchersForResource;
+ }
+ Set<RangerZoneResourceMatcher> tmp = new
HashSet<>();
+ if (smallestList.size() < smaller.size()) {
+
smallestList.stream().filter(smaller::contains).forEach(tmp::add);
+
smallestList.stream().filter(bigger::contains).forEach(tmp::add);
+ } else {
+
smaller.stream().filter(smallestList::contains).forEach(tmp::add);
+ if (smallestList.size() < bigger.size()) {
+
smallestList.stream().filter(bigger::contains).forEach(tmp::add);
+ } else {
+
bigger.stream().filter(smallestList::contains).forEach(tmp::add);
+ }
}
+ smallestList = tmp;
+ }
+ } else {
+ if (CollectionUtils.isEmpty(inheritedZoneMatchers) ||
CollectionUtils.isEmpty(zoneMatchersForResource)) {
+ Set<RangerZoneResourceMatcher> tmp =
CollectionUtils.isEmpty(inheritedZoneMatchers) ? zoneMatchersForResource :
inheritedZoneMatchers;
+ smallestList = resourceKeys.size() == 1 ||
CollectionUtils.isEmpty(tmp) ? tmp : new HashSet<>(tmp);
+ } else {
+ smallestList = new
HashSet<>(zoneMatchersForResource);
+ smallestList.addAll(inheritedZoneMatchers);
}
}
- } else {
- intersection = smallestList;
}
+
if (LOG.isDebugEnabled()) {
- LOG.debug("Resource:[" + resource +"], matched-zones:[" +
intersection +"]");
+ LOG.debug("Resource:[" + resource +"], matched-zones:[" +
smallestList +"]");
}
- if (intersection.size() <= 1) {
+
+ if (CollectionUtils.isEmpty(smallestList) ||
smallestList.size() == 1) {
continue;
}
+ final Set<RangerZoneResourceMatcher> intersection =
smallestList;
+
RangerAccessResourceImpl accessResource = new
RangerAccessResourceImpl();
accessResource.setServiceDef(serviceDef);
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java
b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java
index bbcf82f..33ec2d8 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerServiceDefHelper.java
@@ -289,6 +289,24 @@ public class RangerServiceDefHelper {
return _delegate.isResourceGraphValid();
}
+ public List<String> getOrderedResourceNames(Collection<String>
resourceNames) {
+ final List<String> ret;
+ if (resourceNames != null) {
+ ret = new ArrayList<>();
+ for (String orderedName :
_delegate.getAllOrderedResourceNames()) {
+ for (String resourceName : resourceNames) {
+ if (StringUtils.equals(orderedName,
resourceName) && !ret.contains(resourceName)) {
+ ret.add(resourceName);
+ break;
+ }
+ }
+ }
+ } else {
+ ret = Collections.EMPTY_LIST;
+ }
+ return ret;
+ }
+
/**
* Not designed for public access. Package level only for testability.
*/
@@ -299,6 +317,7 @@ public class RangerServiceDefHelper {
final String _serviceName;
final boolean _checkForCycles;
final boolean _valid;
+ final List<String> _orderedResourceNames;
final static Set<List<RangerResourceDef>>
EMPTY_RESOURCE_HIERARCHY = Collections.unmodifiableSet(new
HashSet<List<RangerResourceDef>>());
@@ -327,6 +346,13 @@ public class RangerServiceDefHelper {
_hierarchies.put(policyType,
EMPTY_RESOURCE_HIERARCHY);
}
}
+
+ if (isValid) {
+ _orderedResourceNames =
buildSortedResourceNames();
+ } else {
+ _orderedResourceNames = new ArrayList<>();
+ }
+
_valid = isValid;
if (LOG.isDebugEnabled()) {
String message = String.format("Found [%d]
resource hierarchies for service [%s] update-date[%s]: %s",
_hierarchies.size(), _serviceName,
@@ -537,6 +563,53 @@ public class RangerServiceDefHelper {
}
return map;
}
+
+ List<String> getAllOrderedResourceNames() {
+ return this._orderedResourceNames;
+ }
+
+ private static class ResourceNameLevel implements
Comparable<ResourceNameLevel> {
+ private String resourceName;
+ private int level;
+
+ ResourceNameLevel(String resourceName, int level) {
+ this.resourceName = resourceName;
+ this.level = level;
+ }
+
+ @Override
+ public int compareTo(ResourceNameLevel other) {
+ return Integer.compare(this.level, other.level);
+ }
+ }
+
+ private List<String> buildSortedResourceNames() {
+ final List<String> ret = new ArrayList<>();
+
+ boolean isValid = true;
+ List<ResourceNameLevel> resourceNameLevels = new
ArrayList<>();
+ for (RangerResourceDef resourceDef :
_serviceDef.getResources()) {
+ String name = resourceDef.getName();
+ Integer level = resourceDef.getLevel();
+ if (name != null && level != null) {
+ ResourceNameLevel resourceNameLevel =
new ResourceNameLevel(name, level);
+
resourceNameLevels.add(resourceNameLevel);
+ } else {
+ LOG.error("Incorrect
resourceDef:[name=" + name + "]");
+ isValid = false;
+ break;
+ }
+ }
+
+ if (isValid) {
+ Collections.sort(resourceNameLevels);
+
+ for (ResourceNameLevel resourceNameLevel :
resourceNameLevels) {
+ ret.add(resourceNameLevel.resourceName);
+ }
+ }
+ return ret;
+ }
}
/**
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java
index 2742312..1909ad4 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java
@@ -37,6 +37,7 @@ import
org.apache.ranger.plugin.contextenricher.RangerContextEnricher;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerPolicyDelta;
import org.apache.ranger.plugin.model.RangerServiceDef;
+import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper;
import org.apache.ranger.plugin.model.validation.RangerZoneResourceMatcher;
import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
import
org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher;
@@ -437,66 +438,70 @@ public class PolicyEngine {
Set<String> ret = null;
if (MapUtils.isNotEmpty(this.resourceZoneTrie)) {
- List<Set<RangerZoneResourceMatcher>> zoneMatchersList = null;
Set<RangerZoneResourceMatcher> smallestList = null;
+ RangerServiceDefHelper serviceDefHelper =
policyRepository.getOptions().getServiceDefHelper();
- for (Map.Entry<String, ?> entry : resource.entrySet()) {
- String resourceDefName
= entry.getKey();
- Object resourceValues
= entry.getValue();
- RangerResourceTrie<RangerZoneResourceMatcher> trie
= resourceZoneTrie.get(resourceDefName);
+ List<String> resourceKeys = resource == null ? new ArrayList<>() :
serviceDefHelper.getOrderedResourceNames(resource.keySet());
+
+ for (String resourceDefName : resourceKeys) {
+ RangerResourceTrie<RangerZoneResourceMatcher> trie =
resourceZoneTrie.get(resourceDefName);
if (trie == null) {
continue;
}
- Set<RangerZoneResourceMatcher> matchedZones =
trie.getEvaluatorsForResource(resourceValues);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("ResourceDefName:[" + resourceDefName + "],
values:[" + resourceValues + "], matched-zones:[" + matchedZones + "]");
- }
+ Object resourceValues = resource.get(resourceDefName);
- if (CollectionUtils.isEmpty(matchedZones)) { // no policies
for this resource, bail out
- zoneMatchersList = null;
- smallestList = null;
+ Set<RangerZoneResourceMatcher> zoneMatchersForResource =
trie.getEvaluatorsForResource(resourceValues);
+ Set<RangerZoneResourceMatcher> inheritedZoneMatchers =
trie.getInheritedEvaluators();
- break;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ResourceDefName:[" + resourceDefName + "],
values:[" + resourceValues + "], matched-zones:[" + zoneMatchersForResource +
"], inherited-zones:[" + inheritedZoneMatchers + "]");
}
- if (smallestList == null) {
- smallestList = matchedZones;
- } else {
- if (zoneMatchersList == null) {
- zoneMatchersList = new ArrayList<>();
-
- zoneMatchersList.add(smallestList);
- }
-
- zoneMatchersList.add(matchedZones);
-
- if (smallestList.size() > matchedZones.size()) {
- smallestList = matchedZones;
- }
- }
- }
- if (smallestList != null) {
- final Set<RangerZoneResourceMatcher> intersection;
-
- if (zoneMatchersList != null) {
- intersection = new HashSet<>(smallestList);
-
- for (Set<RangerZoneResourceMatcher> zoneMatchers :
zoneMatchersList) {
- if (zoneMatchers != smallestList) {
- // remove zones from intersection that are not in
zoneMatchers
- intersection.retainAll(zoneMatchers);
-
- if (CollectionUtils.isEmpty(intersection)) { // if
no zoneMatcher exists, bail out and return empty list
- break;
+ if (smallestList != null) {
+ if (CollectionUtils.isEmpty(inheritedZoneMatchers) &&
CollectionUtils.isEmpty(zoneMatchersForResource)) {
+ smallestList = null;
+ } else if (CollectionUtils.isEmpty(inheritedZoneMatchers))
{
+ smallestList.retainAll(zoneMatchersForResource);
+ } else if
(CollectionUtils.isEmpty(zoneMatchersForResource)) {
+ smallestList.retainAll(inheritedZoneMatchers);
+ } else {
+ Set<RangerZoneResourceMatcher> smaller, bigger;
+ if (zoneMatchersForResource.size() <
inheritedZoneMatchers.size()) {
+ smaller = zoneMatchersForResource;
+ bigger = inheritedZoneMatchers;
+ } else {
+ smaller = inheritedZoneMatchers;
+ bigger = zoneMatchersForResource;
+ }
+ Set<RangerZoneResourceMatcher> tmp = new HashSet<>();
+ if (smallestList.size() < smaller.size()) {
+
smallestList.stream().filter(smaller::contains).forEach(tmp::add);
+
smallestList.stream().filter(bigger::contains).forEach(tmp::add);
+ } else {
+
smaller.stream().filter(smallestList::contains).forEach(tmp::add);
+ if (smallestList.size() < bigger.size()) {
+
smallestList.stream().filter(bigger::contains).forEach(tmp::add);
+ } else {
+
bigger.stream().filter(smallestList::contains).forEach(tmp::add);
}
}
+ smallestList = tmp;
}
} else {
- intersection = smallestList;
+ if (CollectionUtils.isEmpty(inheritedZoneMatchers) ||
CollectionUtils.isEmpty(zoneMatchersForResource)) {
+ Set<RangerZoneResourceMatcher> tmp =
CollectionUtils.isEmpty(inheritedZoneMatchers) ? zoneMatchersForResource :
inheritedZoneMatchers;
+ smallestList = resourceKeys.size() == 1 ||
CollectionUtils.isEmpty(tmp) ? tmp : new HashSet<>(tmp);
+ } else {
+ smallestList = new HashSet<>(zoneMatchersForResource);
+ smallestList.addAll(inheritedZoneMatchers);
+ }
}
+ }
+
+ if (CollectionUtils.isNotEmpty(smallestList)) {
+ final Set<RangerZoneResourceMatcher> intersection =
smallestList;
if (LOG.isDebugEnabled()) {
LOG.debug("Resource:[" + resource + "], matched-zones:[" +
intersection + "]");
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index ffbd908..2655103 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -790,64 +790,71 @@ public class RangerPolicyRepository {
perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG,
"RangerPolicyRepository.getLikelyMatchEvaluators(resource=" +
resource.getAsString() + ")");
}
- Set<String> resourceKeys = resource == null ? null :
resource.getKeys();
+ List<String> resourceKeys = resource == null ?
null :
options.getServiceDefHelper().getOrderedResourceNames(resource.getKeys());
+ Set<RangerPolicyEvaluator> smallestList = null;
- if(CollectionUtils.isNotEmpty(resourceKeys)) {
- Set<RangerPolicyEvaluator> evaluators = null;
- List<Set<RangerPolicyEvaluator>> resourceEvaluatorsSet = null;
- Set<RangerPolicyEvaluator> smallestSet = null;
+ if (CollectionUtils.isNotEmpty(resourceKeys)) {
for (String resourceName : resourceKeys) {
- RangerResourceTrie trie = resourceTrie.get(resourceName);
+ RangerResourceTrie<RangerPolicyEvaluator> trie =
resourceTrie.get(resourceName);
if (trie == null) { // if no trie exists for this resource
level, ignore and continue to next level
continue;
}
- Set<RangerPolicyEvaluator> resourceEvaluators =
trie.getEvaluatorsForResource(resource.getValue(resourceName));
+ Set<RangerPolicyEvaluator> serviceResourceMatchersForResource
= trie.getEvaluatorsForResource(resource.getValue(resourceName));
+ Set<RangerPolicyEvaluator> inheritedResourceMatchers =
trie.getInheritedEvaluators();
- if (CollectionUtils.isEmpty(resourceEvaluators)) { // no
policies for this resource, bail out
- resourceEvaluatorsSet = null;
- smallestSet = null;
- break;
- }
-
- if (smallestSet == null) {
- smallestSet = resourceEvaluators;
- } else {
- if (resourceEvaluatorsSet == null) {
- resourceEvaluatorsSet = new ArrayList<>();
- resourceEvaluatorsSet.add(smallestSet);
+ if (smallestList != null) {
+ if (CollectionUtils.isEmpty(inheritedResourceMatchers) &&
CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
+ smallestList = null;
+ } else if
(CollectionUtils.isEmpty(inheritedResourceMatchers)) {
+
smallestList.retainAll(serviceResourceMatchersForResource);
+ } else if
(CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
+ smallestList.retainAll(inheritedResourceMatchers);
+ } else {
+ Set<RangerPolicyEvaluator> smaller, bigger;
+ if (serviceResourceMatchersForResource.size() <
inheritedResourceMatchers.size()) {
+ smaller = serviceResourceMatchersForResource;
+ bigger = inheritedResourceMatchers;
+ } else {
+ smaller = inheritedResourceMatchers;
+ bigger = serviceResourceMatchersForResource;
+ }
+ Set<RangerPolicyEvaluator> tmp = new HashSet<>();
+ if (smallestList.size() < smaller.size()) {
+
smallestList.stream().filter(smaller::contains).forEach(tmp::add);
+
smallestList.stream().filter(bigger::contains).forEach(tmp::add);
+ } else {
+
smaller.stream().filter(smallestList::contains).forEach(tmp::add);
+ if (smallestList.size() < bigger.size()) {
+
smallestList.stream().filter(bigger::contains).forEach(tmp::add);
+ } else {
+
bigger.stream().filter(smallestList::contains).forEach(tmp::add);
+ }
+ }
+ smallestList = tmp;
}
- resourceEvaluatorsSet.add(resourceEvaluators);
-
- if (smallestSet.size() > resourceEvaluators.size()) {
- smallestSet = resourceEvaluators;
+ } else {
+ if (CollectionUtils.isEmpty(inheritedResourceMatchers) ||
CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
+ Set<RangerPolicyEvaluator> tmp =
CollectionUtils.isEmpty(inheritedResourceMatchers) ?
serviceResourceMatchersForResource : inheritedResourceMatchers;
+ smallestList = resourceKeys.size() == 1 ||
CollectionUtils.isEmpty(tmp) ? tmp : new HashSet<>(tmp);
+ } else {
+ smallestList = new
HashSet<>(serviceResourceMatchersForResource);
+ smallestList.addAll(inheritedResourceMatchers);
}
}
- }
-
- if (resourceEvaluatorsSet != null) {
- evaluators = new HashSet<>(smallestSet);
- for (Set<RangerPolicyEvaluator> resourceEvaluators :
resourceEvaluatorsSet) {
- if (resourceEvaluators != smallestSet) {
- // remove policies from ret that are not in
resourceEvaluators
- evaluators.retainAll(resourceEvaluators);
- if (CollectionUtils.isEmpty(evaluators)) { // if no
policy exists, bail out and return empty list
- evaluators = null;
- break;
- }
- }
+ if (CollectionUtils.isEmpty(smallestList)) {// no tags for
this resource, bail out
+ smallestList = null;
+ break;
}
- } else {
- evaluators = smallestSet;
}
+ }
- if (evaluators != null) {
- ret = new ArrayList<>(evaluators);
- ret.sort(RangerPolicyEvaluator.EVAL_ORDER_COMPARATOR);
- }
+ if (smallestList != null) {
+ ret = new ArrayList<>(smallestList);
+ ret.sort(RangerPolicyEvaluator.EVAL_ORDER_COMPARATOR);
}
RangerPerfTracer.logAlways(perf);
@@ -1320,10 +1327,7 @@ public class RangerPolicyRepository {
private void removeEvaluatorFromTrie(RangerPolicyEvaluator oldEvaluator,
RangerResourceTrie<RangerPolicyEvaluator> trie, String resourceDefName) {
if (oldEvaluator != null) {
- RangerPolicy.RangerPolicyResource resource =
oldEvaluator.getPolicyResource().get(resourceDefName);
- if (resource != null) {
- trie.delete(resource, oldEvaluator);
- }
+ trie.delete(oldEvaluator.getPolicyResource().get(resourceDefName),
oldEvaluator);
}
}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
index 4428503..b41da47 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java
@@ -57,6 +57,7 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
private final String wildcardChars;
private final TrieNode<T> root;
private final boolean isOptimizedForRetrieval;
+ private Set<T> inheritedEvaluators;
public RangerResourceTrie(RangerServiceDef.RangerResourceDef resourceDef,
List<T> evaluators) {
this(resourceDef, evaluators, true, null);
@@ -73,6 +74,7 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
this.optIgnoreCase = other.optIgnoreCase;
this.optWildcard = other.optWildcard;
this.wildcardChars = other.wildcardChars;
+ this.inheritedEvaluators = other.inheritedEvaluators != null ? new
HashSet<>(other.inheritedEvaluators) : null;
this.isOptimizedForRetrieval = false;
this.root = copyTrieSubtree(other.root, null);
@@ -159,6 +161,10 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
}
}
+ public Set<T> getInheritedEvaluators() {
+ return inheritedEvaluators;
+ }
+
public Set<T> getEvaluatorsForResource(Object resource) {
if (resource instanceof String) {
return getEvaluatorsForResource((String) resource);
@@ -186,11 +192,11 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
if (resource == null) {
if (evaluator.isAncestorOf(resourceDef)) {
- root.addWildcardEvaluator(evaluator);
+ addInheritedEvaluator(evaluator);
}
} else {
if (resource.getIsExcludes()) {
- root.addWildcardEvaluator(evaluator);
+ addInheritedEvaluator(evaluator);
} else {
if (CollectionUtils.isNotEmpty(resource.getValues())) {
for (String value : resource.getValues()) {
@@ -216,11 +222,13 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_INIT_LOG,
"RangerResourceTrie.delete(name=" + resource + ")");
}
- boolean isRemoved = false;
- if (resource.getIsExcludes()) {
- isRemoved = root.removeWildcardEvaluator(evaluator);
- }
- if (!isRemoved) {
+ if (resource == null) {
+ if (evaluator.isAncestorOf(resourceDef)) {
+ removeInheritedEvaluator(evaluator);
+ }
+ } else if (resource.getIsExcludes()) {
+ removeInheritedEvaluator(evaluator);
+ } else {
for (String value : resource.getValues()) {
TrieNode<T> node = getNodeForResource(value);
if (node != null) {
@@ -247,6 +255,23 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
return root;
}
+ private void addInheritedEvaluator(T evaluator) {
+ if (inheritedEvaluators == null) {
+ inheritedEvaluators = new HashSet<>();
+ }
+
+ inheritedEvaluators.add(evaluator);
+ }
+
+ private void removeInheritedEvaluator(T evaluator) {
+ if (CollectionUtils.isNotEmpty(inheritedEvaluators) &&
inheritedEvaluators.contains(evaluator)) {
+ inheritedEvaluators.remove(evaluator);
+ if (CollectionUtils.isEmpty(inheritedEvaluators)) {
+ inheritedEvaluators = null;
+ }
+ }
+ }
+
private TrieNode<T> copyTrieSubtree(final TrieNode<T> source, final
TrieNode<T> parent) {
if (TRACE_LOG.isTraceEnabled()) {
StringBuilder sb = new StringBuilder();
@@ -339,14 +364,14 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
if (policyResource == null) {
if (evaluator.isAncestorOf(resourceDef)) {
- ret.addWildcardEvaluator(evaluator);
+ addInheritedEvaluator(evaluator);
}
continue;
}
if (policyResource.getIsExcludes()) {
- ret.addWildcardEvaluator(evaluator);
+ addInheritedEvaluator(evaluator);
} else {
RangerResourceMatcher resourceMatcher =
evaluator.getResourceMatcher(resourceName);
@@ -810,7 +835,7 @@ public class RangerResourceTrie<T extends
RangerPolicyResourceEvaluator> {
}
for (Map.Entry<Character, TrieNode<U>> entry :
children.entrySet()) {
- TrieNode child = entry.getValue();
+ TrieNode<U> child = entry.getValue();
child.populateTrieData(trieData);
}
diff --git
a/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java
b/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java
index 83b39ed..799787c 100644
---
a/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java
+++
b/agents-common/src/test/java/org/apache/ranger/plugin/contextenricher/TestTagEnricher.java
@@ -96,6 +96,7 @@ public class TestTagEnricher {
tagEnricher.setServiceName(testCase.serviceName);
tagEnricher.setServiceDef(testCase.serviceDef);
+ tagEnricher.init();
tagEnricher.setServiceTags(serviceTags);
List<String> expectedTags = new ArrayList<>();
diff --git
a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngineComparison.java
b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngineComparison.java
index e204983..6f644cc 100644
---
a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngineComparison.java
+++
b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngineComparison.java
@@ -138,11 +138,15 @@ public class TestPolicyEngineComparison {
myTagEnricher.setAppId("test-compare-my-tags");
myTagEnricher.setServiceDef(myServicePolicies.getServiceDef());
myTagEnricher.setServiceName(myServiceTags.getServiceName());
+ myTagEnricher.init();
+
myTagEnricher.setServiceTags(myServiceTags);
otherTagEnricher.setAppId("test-compare-other-tags");
otherTagEnricher.setServiceDef(myServicePolicies.getServiceDef());
otherTagEnricher.setServiceName(otherServiceTags.getServiceName());
+ otherTagEnricher.init();
+
otherTagEnricher.setServiceTags(otherServiceTags);
isTagsEqual = TestPolicyEngine.compare(myTagEnricher,
otherTagEnricher) && TestPolicyEngine.compare(otherTagEnricher, myTagEnricher);
