This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch ranger-2.6 in repository https://gitbox.apache.org/repos/asf/ranger.git
commit a740901fb4dad5bc0a4329911d9861b425993ce4 Author: Abhay Kulkarni <[email protected]> AuthorDate: Thu Aug 15 09:57:56 2024 -0700 RANGER-4905:Reduce memory needed to create Ranger policy engine (cherry picked from commit c0480ed72656d56cf2e5ab03bbea37e372363081) --- .../plugin/contextenricher/RangerTagEnricher.java | 13 ++-- .../validation/RangerSecurityZoneValidator.java | 2 +- .../validation/RangerZoneResourceMatcher.java | 8 ++- .../ranger/plugin/policyengine/PolicyEngine.java | 1 + .../plugin/policyengine/RangerPluginContext.java | 75 ++++++++++++++++++++-- .../policyengine/RangerPolicyEngineOptions.java | 10 +++ .../policyengine/RangerSecurityZoneMatcher.java | 2 +- .../RangerAbstractPolicyEvaluator.java | 1 + .../RangerDefaultPolicyResourceMatcher.java | 51 ++++++++++----- .../RangerPolicyResourceMatcher.java | 3 + .../org/apache/ranger/sizing/RangerMemSizing.java | 22 ++++--- 11 files changed, 146 insertions(+), 42 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java index 2fa24eba6..a8fbc0215 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java @@ -29,11 +29,7 @@ import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceResource; import org.apache.ranger.plugin.model.RangerTag; import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper; -import org.apache.ranger.plugin.policyengine.RangerAccessRequest; -import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; -import org.apache.ranger.plugin.policyengine.RangerAccessResource; -import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; -import org.apache.ranger.plugin.policyengine.RangerResourceTrie; +import org.apache.ranger.plugin.policyengine.*; import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; import org.apache.ranger.plugin.util.DownloadTrigger; @@ -437,7 +433,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { for (ListIterator<RangerServiceResource> iter = serviceResources.listIterator(); iter.hasNext(); ) { RangerServiceResource serviceResource = iter.next(); - RangerServiceResourceMatcher serviceResourceMatcher = createRangerServiceResourceMatcher(serviceResource, serviceDefHelper, hierarchies); + RangerServiceResourceMatcher serviceResourceMatcher = createRangerServiceResourceMatcher(serviceResource, serviceDefHelper, hierarchies, getPluginContext()); if (serviceResourceMatcher != null) { resourceMatchers.add(serviceResourceMatcher); @@ -484,7 +480,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { if (removedOldServiceResource) { if (!StringUtils.isEmpty(serviceResource.getResourceSignature())) { - RangerServiceResourceMatcher resourceMatcher = createRangerServiceResourceMatcher(serviceResource, serviceDefHelper, hierarchies); + RangerServiceResourceMatcher resourceMatcher = createRangerServiceResourceMatcher(serviceResource, serviceDefHelper, hierarchies, getPluginContext()); if (resourceMatcher != null) { for (RangerServiceDef.RangerResourceDef resourceDef : serviceDef.getResources()) { @@ -613,7 +609,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { return ret; } - static public RangerServiceResourceMatcher createRangerServiceResourceMatcher(RangerServiceResource serviceResource, RangerServiceDefHelper serviceDefHelper, ResourceHierarchies hierarchies) { + static public RangerServiceResourceMatcher createRangerServiceResourceMatcher(RangerServiceResource serviceResource, RangerServiceDefHelper serviceDefHelper, ResourceHierarchies hierarchies, RangerPluginContext pluginContext) { if (LOG.isDebugEnabled()) { LOG.debug("==> createRangerServiceResourceMatcher(serviceResource=" + serviceResource + ")"); @@ -644,6 +640,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { matcher.setServiceDef(serviceDefHelper.getServiceDef()); matcher.setPolicyResources(serviceResource.getResourceElements(), policyType); + matcher.setPluginContext(pluginContext); if (LOG.isDebugEnabled()) { LOG.debug("RangerTagEnricher.setServiceTags() - Initializing matcher with (resource=" + serviceResource diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerSecurityZoneValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerSecurityZoneValidator.java index 360426198..b88ac21a6 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerSecurityZoneValidator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerSecurityZoneValidator.java @@ -354,7 +354,7 @@ public class RangerSecurityZoneValidator extends RangerValidator { policyResources.put(resourceDefName, policyResource); } - RangerZoneResourceMatcher matcher = new RangerZoneResourceMatcher(zone.getName(), policyResources, serviceDefHelper); + RangerZoneResourceMatcher matcher = new RangerZoneResourceMatcher(zone.getName(), policyResources, serviceDefHelper, null); matchers.add(matcher); resourceNames.addAll(policyResources.keySet()); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java index bf4247660..1a8a867a0 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/validation/RangerZoneResourceMatcher.java @@ -21,6 +21,7 @@ package org.apache.ranger.plugin.model.validation; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.policyengine.RangerPluginContext; import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; import org.apache.ranger.plugin.policyresourcematcher.RangerResourceEvaluator; @@ -41,17 +42,18 @@ public class RangerZoneResourceMatcher implements RangerResourceEvaluator { private final RangerPolicyResourceMatcher policyResourceMatcher; private RangerServiceDef.RangerResourceDef leafResourceDef; - public RangerZoneResourceMatcher(final String securityZoneName, final Map<String, RangerPolicy.RangerPolicyResource> policyResource, final RangerServiceDef serviceDef) { - this(securityZoneName, policyResource, new RangerServiceDefHelper(serviceDef)); + public RangerZoneResourceMatcher(final String securityZoneName, final Map<String, RangerPolicy.RangerPolicyResource> policyResource, final RangerServiceDef serviceDef, RangerPluginContext pluginContext) { + this(securityZoneName, policyResource, new RangerServiceDefHelper(serviceDef), pluginContext); } - public RangerZoneResourceMatcher(final String securityZoneName, final Map<String, RangerPolicy.RangerPolicyResource> policyResource, final RangerServiceDefHelper serviceDefHelper) { + public RangerZoneResourceMatcher(final String securityZoneName, final Map<String, RangerPolicy.RangerPolicyResource> policyResource, final RangerServiceDefHelper serviceDefHelper, RangerPluginContext pluginContext) { final RangerServiceDef serviceDef = serviceDefHelper.getServiceDef(); final Collection<String> resourceKeys = policyResource.keySet(); final RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher(); matcher.setServiceDef(serviceDef); matcher.setServiceDefHelper(serviceDefHelper); + matcher.setPluginContext(pluginContext); boolean found = false; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java index 704434b8e..2de3cfa0d 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java @@ -197,6 +197,7 @@ public class PolicyEngine { } normalizeServiceDefs(servicePolicies); + pluginContext.cleanResourceMatchers(); this.pluginContext = pluginContext; this.lock = new RangerReadWriteLock(isUseReadWriteLock); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPluginContext.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPluginContext.java index 5f086ed49..8a3e43e48 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPluginContext.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPluginContext.java @@ -23,18 +23,26 @@ import org.apache.commons.lang.StringUtils; import org.apache.ranger.admin.client.RangerAdminClient; import org.apache.ranger.admin.client.RangerAdminRESTClient; import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; import org.apache.ranger.plugin.service.RangerAuthContext; import org.apache.ranger.plugin.service.RangerAuthContextListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.locks.ReentrantReadWriteLock; + public class RangerPluginContext { private static final Logger LOG = LoggerFactory.getLogger(RangerPluginContext.class); - private final RangerPluginConfig config; - private RangerAuthContext authContext; - private RangerAuthContextListener authContextListener; - private RangerAdminClient adminClient; + private final RangerPluginConfig config; + private RangerAuthContext authContext; + private RangerAuthContextListener authContextListener; + private RangerAdminClient adminClient; + private final Map<String, Map<RangerPolicy.RangerPolicyResource, RangerResourceMatcher>> resourceMatchers = new HashMap<>(); + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); // fair lock public RangerPluginContext(RangerPluginConfig config) { @@ -53,6 +61,65 @@ public class RangerPluginContext { public RangerAuthContext getAuthContext() { return authContext; } + public RangerResourceMatcher getResourceMatcher(String resourceDefName, RangerPolicy.RangerPolicyResource resource) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> getResourceMatcher(resourceDefName={}, resource={})", resourceDefName, resource); + } + RangerResourceMatcher ret = null; + + try { + lock.readLock().lock(); + + Map<RangerPolicy.RangerPolicyResource, RangerResourceMatcher> matchersForResource = resourceMatchers.get(resourceDefName); + + if (matchersForResource != null) { + ret = matchersForResource.get(resource); + } + } finally { + lock.readLock().unlock(); + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== getResourceMatcher(resourceDefName={}, resource={}) : ret={}", resourceDefName, resource, ret); + } + + return ret; + } + + public void setResourceMatcher(String resourceDefName, RangerPolicy.RangerPolicyResource resource, RangerResourceMatcher matcher) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> setResourceMatcher(resourceDefName={}, resource={}, matcher={})", resourceDefName, resource, matcher); + } + if (config != null && config.getPolicyEngineOptions().enableResourceMatcherReuse) { + try { + lock.writeLock().lock(); + + Map<RangerPolicy.RangerPolicyResource, RangerResourceMatcher> matchersForResource = resourceMatchers.computeIfAbsent(resourceDefName, k -> new HashMap<>()); + matchersForResource.put(resource, matcher); + } finally { + lock.writeLock().unlock(); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== setResourceMatcher(resourceDefName={}, resource={}, matcher={})", resourceDefName, resource, matcher); + } + } + + void cleanResourceMatchers() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> cleanResourceMatchers()"); + } + try { + lock.writeLock().lock(); + + resourceMatchers.clear(); + } finally { + lock.writeLock().unlock(); + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== cleanResourceMatchers()"); + } + } + public void setAuthContext(RangerAuthContext authContext) { this.authContext = authContext; } public void setAuthContextListener(RangerAuthContextListener authContextListener) { this.authContextListener = authContextListener; } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java index 800721e53..0828949a4 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java @@ -36,6 +36,7 @@ public class RangerPolicyEngineOptions { public boolean evaluateDelegateAdminOnly = false; public boolean enableTagEnricherWithLocalRefresher = false; public boolean enableUserStoreEnricherWithLocalRefresher = false; + public boolean enableResourceMatcherReuse = true; @Deprecated public boolean disableAccessEvaluationWithPolicyACLSummary = true; public boolean optimizeTrieForRetrieval = false; @@ -61,6 +62,7 @@ public class RangerPolicyEngineOptions { this.evaluateDelegateAdminOnly = other.evaluateDelegateAdminOnly; this.enableTagEnricherWithLocalRefresher = other.enableTagEnricherWithLocalRefresher; this.enableUserStoreEnricherWithLocalRefresher = other.enableUserStoreEnricherWithLocalRefresher; + this.enableResourceMatcherReuse = other.enableResourceMatcherReuse; this.optimizeTrieForRetrieval = other.optimizeTrieForRetrieval; this.disableRoleResolution = other.disableRoleResolution; this.serviceDefHelper = null; @@ -79,6 +81,7 @@ public class RangerPolicyEngineOptions { disableUserStoreRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.userstore.retriever", false); cacheAuditResults = conf.getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", true); + enableResourceMatcherReuse = conf.getBoolean(propertyPrefix + ".policyengine.option.enable.resourcematcher.reuse", true); if (!disableTrieLookupPrefilter) { cacheAuditResults = false; @@ -109,6 +112,7 @@ public class RangerPolicyEngineOptions { enableUserStoreEnricherWithLocalRefresher = false; optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false); disableRoleResolution = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.role.resolution", true); + enableResourceMatcherReuse = conf.getBoolean(propertyPrefix + ".policyengine.option.enable.resourcematcher.reuse", true); } public void configureDelegateAdmin(Configuration conf, String propertyPrefix) { @@ -120,6 +124,7 @@ public class RangerPolicyEngineOptions { disableTagRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tag.retriever", true); disableUserStoreRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.userstore.retriever", true); optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false); + enableResourceMatcherReuse = conf.getBoolean(propertyPrefix + ".policyengine.option.enable.resourcematcher.reuse", true); cacheAuditResults = false; evaluateDelegateAdminOnly = true; @@ -145,6 +150,7 @@ public class RangerPolicyEngineOptions { optimizeTrieForSpace = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.space", false); optimizeTagTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.tag.trie.for.retrieval", false); optimizeTagTrieForSpace = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.tag.trie.for.space", true); + enableResourceMatcherReuse = conf.getBoolean(propertyPrefix + ".policyengine.option.enable.resourcematcher.reuse", true); } public RangerServiceDefHelper getServiceDefHelper() { @@ -181,6 +187,7 @@ public class RangerPolicyEngineOptions { && this.optimizeTrieForSpace == that.optimizeTrieForSpace && this.optimizeTagTrieForRetrieval == that.optimizeTagTrieForRetrieval && this.optimizeTagTrieForSpace == that.optimizeTagTrieForSpace + && this.enableResourceMatcherReuse == that.enableResourceMatcherReuse ; } return ret; @@ -221,6 +228,8 @@ public class RangerPolicyEngineOptions { ret *= 2; ret += optimizeTagTrieForSpace ? 1 : 0; ret *= 2; + ret += enableResourceMatcherReuse ? 1 : 0; + ret *= 2; return ret; } @@ -244,6 +253,7 @@ public class RangerPolicyEngineOptions { ", optimizeTrieForSpace: " + optimizeTrieForSpace + ", optimizeTagTrieForRetrieval: " + optimizeTagTrieForRetrieval + ", optimizeTagTrieForSpace: " + optimizeTagTrieForSpace + + ", enableResourceMatcherReuse: " + enableResourceMatcherReuse + " }"; } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerSecurityZoneMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerSecurityZoneMatcher.java index 822bb3902..0d44f7109 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerSecurityZoneMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerSecurityZoneMatcher.java @@ -171,7 +171,7 @@ public class RangerSecurityZoneMatcher { policyResources.put(resourceDefName, new RangerPolicyResource(resourceValues, false, isRecursive)); } - matchers.add(new RangerZoneResourceMatcher(zoneName, policyResources, serviceDef)); + matchers.add(new RangerZoneResourceMatcher(zoneName, policyResources, serviceDef, pluginContext)); if (LOG.isDebugEnabled()) { LOG.debug("Built matcher for resource:[{}] in zone:[{}]", resource, zoneName); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java index 3c79d81a6..a72656287 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java @@ -542,6 +542,7 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu this.resourceMatcher.setPolicyResources(resource, policyType); this.resourceMatcher.setServiceDef(serviceDef); this.resourceMatcher.setServiceDefHelper(serviceDefHelper); + this.resourceMatcher.setPluginContext(pluginContext); this.resourceMatcher.init(); } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java index f16157ce6..0c377b357 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerDefaultPolicyResourceMatcher.java @@ -37,6 +37,7 @@ import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper; import org.apache.ranger.plugin.policyengine.RangerAccessRequest.ResourceElementMatchingScope; import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; +import org.apache.ranger.plugin.policyengine.RangerPluginContext; import org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher; import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; import org.apache.ranger.plugin.util.RangerPerfTracer; @@ -60,6 +61,7 @@ public class RangerDefaultPolicyResourceMatcher implements RangerPolicyResourceM private List<RangerResourceDef> validResourceHierarchy; private boolean isInitialized = false; private RangerServiceDefHelper serviceDefHelper; + private RangerPluginContext pluginContext = null; private final boolean forceEnableWildcardMatch; @@ -113,6 +115,9 @@ public class RangerDefaultPolicyResourceMatcher implements RangerPolicyResourceM this.serviceDefHelper = serviceDefHelper; } + @Override + public void setPluginContext(RangerPluginContext pluginContext) { this.pluginContext = pluginContext; } + public int getPolicyType() { return policyType; } public RangerServiceDefHelper getServiceDefHelper() { @@ -812,29 +817,41 @@ public class RangerDefaultPolicyResourceMatcher implements RangerPolicyResourceM String resName = resourceDef.getName(); String clsName = resourceDef.getMatcher(); - if (!StringUtils.isEmpty(clsName)) { - try { - @SuppressWarnings("unchecked") - Class<RangerResourceMatcher> matcherClass = (Class<RangerResourceMatcher>) Class.forName(clsName); + if (pluginContext != null) { + ret = pluginContext.getResourceMatcher(resName, resource); + } + + if (ret == null) { + if (!StringUtils.isEmpty(clsName)) { + try { + @SuppressWarnings("unchecked") Class<RangerResourceMatcher> matcherClass = (Class<RangerResourceMatcher>) Class.forName(clsName); - ret = matcherClass.newInstance(); - } catch (Exception excp) { - LOG.error("failed to instantiate resource matcher '" + clsName + "' for '" + resName + "'. Default resource matcher will be used", excp); + ret = matcherClass.newInstance(); + } catch (Exception excp) { + LOG.error("failed to instantiate resource matcher '" + clsName + "' for '" + resName + "'. Default resource matcher will be used", excp); + } } - } - if (ret == null) { - ret = new RangerDefaultResourceMatcher(); - } + if (ret == null) { + ret = new RangerDefaultResourceMatcher(); + } - if (forceEnableWildcardMatch && !Boolean.parseBoolean(resourceDef.getMatcherOptions().get(OPTION_WILD_CARD))) { - resourceDef = serviceDefHelper.getWildcardEnabledResourceDef(resourceDef.getName(), policyType); - } + if (forceEnableWildcardMatch && !Boolean.parseBoolean(resourceDef.getMatcherOptions().get(OPTION_WILD_CARD))) { + resourceDef = serviceDefHelper.getWildcardEnabledResourceDef(resourceDef.getName(), policyType); + } - ret.setResourceDef(resourceDef); - ret.setPolicyResource(resource); - ret.init(); + ret.setResourceDef(resourceDef); + ret.setPolicyResource(resource); + ret.init(); + if (pluginContext != null) { + pluginContext.setResourceMatcher(resName, resource, ret); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Did not create a fresh matcher - used matcher from pluginContext"); + } + } } else { LOG.error("RangerDefaultPolicyResourceMatcher: RangerResourceDef is null"); } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java index e1cd89b70..ad6869ad0 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyresourcematcher/RangerPolicyResourceMatcher.java @@ -28,6 +28,7 @@ import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper; import org.apache.ranger.plugin.policyengine.RangerAccessRequest.ResourceElementMatchingScope; import org.apache.ranger.plugin.policyengine.RangerAccessResource; +import org.apache.ranger.plugin.policyengine.RangerPluginContext; import org.apache.ranger.plugin.resourcematcher.RangerResourceMatcher; public interface RangerPolicyResourceMatcher { @@ -48,6 +49,8 @@ public interface RangerPolicyResourceMatcher { void setServiceDefHelper(RangerServiceDefHelper serviceDefHelper); + void setPluginContext(RangerPluginContext pluginContext); + RangerServiceDef getServiceDef(); RangerResourceMatcher getResourceMatcher(String resourceName); diff --git a/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java b/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java index ee9d8b47f..7dbcbb67e 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java +++ b/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java @@ -83,6 +83,7 @@ public class RangerMemSizing { private final boolean deDup; private final boolean deDupStrings; private final String optimizationMode; + private final boolean reuseResourceMatchers; private final PrintStream out; public RangerMemSizing(CommandLine cmdLine) { @@ -97,6 +98,7 @@ public class RangerMemSizing { this.deDup = Boolean.parseBoolean(cmdLine.getOptionValue("d", "true")); this.deDupStrings = this.deDup; this.optimizationMode = StringUtils.startsWithIgnoreCase(cmdLine.getOptionValue('o', "space"), "s") ? OPT_MODE_SPACE : OPT_MODEL_RETRIEVAL; + this.reuseResourceMatchers = Boolean.parseBoolean(cmdLine.getOptionValue('m', "true")); } public void run() { @@ -127,31 +129,32 @@ public class RangerMemSizing { out.println(); out.println("Parameters:"); if (policies != null) { - out.println(" Policies: file=" + policyFile + ", size=" + new File(policyFile).length() + ", " + toSummaryStr(policies)); + out.println(" Policies: file=" + policyFile + ", size=" + new File(policyFile).length() + ", " + toSummaryStr(policies)); } if (tags != null) { - out.println(" Tags: file=" + tagFile + ", size=" + new File(tagFile).length() + ", " + toSummaryStr(tags)); + out.println(" Tags: file=" + tagFile + ", size=" + new File(tagFile).length() + ", " + toSummaryStr(tags)); } if (roles != null) { - out.println(" Roles: file=" + rolesFile + ", size=" + new File(rolesFile).length() + ", " + toSummaryStr(roles)); + out.println(" Roles: file=" + rolesFile + ", size=" + new File(rolesFile).length() + ", " + toSummaryStr(roles)); } if (userStore != null) { - out.println(" UserStore: file=" + userStoreFile + ", size=" + new File(userStoreFile).length() + ", " + toSummaryStr(userStore)); + out.println(" UserStore: file=" + userStoreFile + ", size=" + new File(userStoreFile).length() + ", " + toSummaryStr(userStore)); } if (genRequestsFile != null) { - out.println(" GenReq: file=" + genRequestsFile + ", requestCount=" + genReqCount); + out.println(" GenReq: file=" + genRequestsFile + ", requestCount=" + genReqCount); } if (evalRequestsFile != null) { - out.println(" EvalReq: file=" + evalRequestsFile + ", requestCount=" + evalReqCount + ", avgTimeTaken=" + evalAvgTimeNs + "ns, clientCount=" + evalClientsCount); + out.println(" EvalReq: file=" + evalRequestsFile + ", requestCount=" + evalReqCount + ", avgTimeTaken=" + evalAvgTimeNs + "ns, clientCount=" + evalClientsCount); } - out.println(" DeDup: " + deDup); - out.println(" OptMode: " + optimizationMode); + out.println(" DeDup: " + deDup); + out.println(" OptMode: " + optimizationMode); + out.println(" ReuseMatchers: " + reuseResourceMatchers); out.println(); out.println("Results:"); @@ -511,6 +514,7 @@ public class RangerMemSizing { Option evalClients = new Option("c", "evalClients", true, "eval clients count"); Option gdsInfo = new Option("g", "gdsInfo", true, "gdsInfo file"); Option optimizeMode = new Option("o", "optMode", true, "optimization mode: space|retrieval"); + Option reuseResourceMatchers = new Option("m", "reuseResourceMatchers", true, "reuse resource matchers: true|false"); Options options = new Options(); @@ -525,6 +529,7 @@ public class RangerMemSizing { options.addOption(gdsInfo); options.addOption(deDup); options.addOption(optimizeMode); + options.addOption(reuseResourceMatchers); try { CommandLine cmdLine = new DefaultParser().parse(options, args); @@ -562,6 +567,7 @@ public class RangerMemSizing { ret.optimizeTrieForRetrieval = !ret.optimizeTrieForSpace; ret.optimizeTagTrieForSpace = ret.optimizeTrieForSpace; ret.optimizeTagTrieForRetrieval = ret.optimizeTrieForRetrieval; + ret.enableResourceMatcherReuse = reuseResourceMatchers; return ret; }
