RANGER-836: commit id 4e2e83eef660fae5287d4d1dc7bbea68015445ee Signed-off-by: Madhan Neethiraj <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/d3a2964f Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/d3a2964f Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/d3a2964f Branch: refs/heads/ranger-0.5 Commit: d3a2964fb7001ddc526d2d3eff9445ebc736c4d2 Parents: 36fbb78 Author: Abhay Kulkarni <[email protected]> Authored: Sun Mar 20 10:50:21 2016 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Sun Mar 20 11:16:49 2016 -0700 ---------------------------------------------------------------------- .../policyengine/RangerPolicyEngineImpl.java | 25 ++ .../policyengine/RangerPolicyEngineOptions.java | 2 +- .../policyengine/RangerPolicyRepository.java | 91 +++++- .../policyevaluator/RangerPolicyEvaluator.java | 2 +- .../ranger/plugin/service/RangerBasePlugin.java | 2 +- .../ranger/plugin/store/ServiceStore.java | 5 + .../plugin/store/file/ServiceFileStore.java | 17 +- .../plugin/store/rest/ServiceRESTStore.java | 16 + .../org/apache/ranger/biz/ServiceDBStore.java | 93 ++++-- .../common/RangerServicePoliciesCache.java | 298 +++++++++++++++++++ .../org/apache/ranger/rest/ServiceREST.java | 2 +- 11 files changed, 515 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d3a2964f/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index c276d5a..55ae785 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@ -20,8 +20,10 @@ package org.apache.ranger.plugin.policyengine; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.contextenricher.RangerContextEnricher; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerServiceDef; @@ -45,6 +47,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { private static final Log PERF_POLICYENGINE_AUDIT_LOG = RangerPerfTracer.getPerfLogger("policyengine.audit"); private static final Log PERF_CONTEXTENRICHER_REQUEST_LOG = RangerPerfTracer.getPerfLogger("contextenricher.request"); + private static final int MAX_POLICIES_FOR_CACHE_TYPE_EVALUATOR = 500; + private final RangerPolicyRepository policyRepository; @@ -67,6 +71,27 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { options = new RangerPolicyEngineOptions(); } + if(StringUtils.isBlank(options.evaluatorType) || StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_AUTO)) { + + String serviceType = servicePolicies.getServiceDef().getName(); + String propertyName = "ranger.plugin." + serviceType + ".policyengine.evaluator.auto.maximum.policycount.for.cache.type"; + + int thresholdForUsingOptimizedEvaluator = RangerConfiguration.getInstance().getInt(propertyName, MAX_POLICIES_FOR_CACHE_TYPE_EVALUATOR); + + int servicePoliciesCount = servicePolicies.getPolicies().size(); + + if (servicePoliciesCount > thresholdForUsingOptimizedEvaluator) { + options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; + } else { + options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED; + } + } else if (StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED)) { + options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED; + } else { + // All other cases + options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; + } + policyRepository = new RangerPolicyRepository(servicePolicies, options); RangerPerfTracer.log(perf); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d3a2964f/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java index a5c1dfb..3289661 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java @@ -23,7 +23,7 @@ import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; public class RangerPolicyEngineOptions { - public String evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED; + public String evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_AUTO; public boolean cacheAuditResults = true; public boolean disableContextEnrichers = false; public boolean disableCustomConditions = false; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d3a2964f/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index 1f422c5..595c324 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -49,8 +49,8 @@ public class RangerPolicyRepository { private final RangerServiceDef serviceDef; private final List<RangerPolicy> policies; private final long policyVersion; - private final List<RangerContextEnricher> contextEnrichers; - private final List<RangerPolicyEvaluator> policyEvaluators; + private List<RangerContextEnricher> contextEnrichers; + private List<RangerPolicyEvaluator> policyEvaluators; private final Map<String, Boolean> accessAuditCache; private static int RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE = 64*1024; @@ -130,6 +130,85 @@ public class RangerPolicyRepository { return policyEvaluators; } + public static boolean isDelegateAdminPolicy(RangerPolicy policy) { + boolean ret = false; + + ret = hasDelegateAdminItems(policy.getPolicyItems()); + + return ret; + } + + private static boolean hasDelegateAdminItems(List<RangerPolicy.RangerPolicyItem> items) { + boolean ret = false; + + if (CollectionUtils.isNotEmpty(items)) { + for (RangerPolicy.RangerPolicyItem item : items) { + if(item.getDelegateAdmin()) { + ret = true; + + break; + } + } + } + return ret; + } + + private static boolean skipBuildingPolicyEvaluator(RangerPolicy policy, RangerPolicyEngineOptions options) { + boolean ret = false; + if (!policy.getIsEnabled()) { + ret = true; + } + return ret; + } + + private void init(RangerPolicyEngineOptions options) { + + List<RangerPolicyEvaluator> policyEvaluators = new ArrayList<RangerPolicyEvaluator>(); + + for (RangerPolicy policy : policies) { + if (skipBuildingPolicyEvaluator(policy, options)) { + continue; + } + + RangerPolicyEvaluator evaluator = buildPolicyEvaluator(policy, serviceDef, options); + + if (evaluator != null) { + policyEvaluators.add(evaluator); + } + } + Collections.sort(policyEvaluators); + this.policyEvaluators = Collections.unmodifiableList(policyEvaluators); + + List<RangerContextEnricher> contextEnrichers = new ArrayList<RangerContextEnricher>(); + if (CollectionUtils.isNotEmpty(this.policyEvaluators)) { + if (!options.disableContextEnrichers && !CollectionUtils.isEmpty(serviceDef.getContextEnrichers())) { + for (RangerServiceDef.RangerContextEnricherDef enricherDef : serviceDef.getContextEnrichers()) { + if (enricherDef == null) { + continue; + } + + RangerContextEnricher contextEnricher = buildContextEnricher(enricherDef); + + if (contextEnricher != null) { + contextEnrichers.add(contextEnricher); + } + } + } + } + this.contextEnrichers = Collections.unmodifiableList(contextEnrichers); + + if(LOG.isDebugEnabled()) { + LOG.debug("policy evaluation order: " + this.policyEvaluators.size() + " policies"); + + int order = 0; + for(RangerPolicyEvaluator policyEvaluator : this.policyEvaluators) { + RangerPolicy policy = policyEvaluator.getPolicy(); + + LOG.debug("policy evaluation order: #" + (++order) + " - policy id=" + policy.getId() + "; name=" + policy.getName() + "; evalOrder=" + policyEvaluator.getEvalOrder()); + } + } + } + private RangerContextEnricher buildContextEnricher(RangerServiceDef.RangerContextEnricherDef enricherDef) { if(LOG.isDebugEnabled()) { LOG.debug("==> RangerPolicyRepository.buildContextEnricher(" + enricherDef + ")"); @@ -178,14 +257,10 @@ public class RangerPolicyRepository { scrubPolicy(policy); RangerPolicyEvaluator ret = null; - if(StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_DEFAULT)) { - ret = new RangerDefaultPolicyEvaluator(); - } else if(StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED)) { - ret = new RangerOptimizedPolicyEvaluator(); - } else if(StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED)) { + if(StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED)) { ret = new RangerCachedPolicyEvaluator(); } else { - ret = new RangerDefaultPolicyEvaluator(); + ret = new RangerOptimizedPolicyEvaluator(); } ret.init(policy, serviceDef, options); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d3a2964f/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java index 624ff1c..e6ec2ad 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java @@ -32,7 +32,7 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; public interface RangerPolicyEvaluator extends Comparable<RangerPolicyEvaluator> { - public static final String EVALUATOR_TYPE_DEFAULT = "default"; + public static final String EVALUATOR_TYPE_AUTO = "auto"; public static final String EVALUATOR_TYPE_OPTIMIZED = "optimized"; public static final String EVALUATOR_TYPE_CACHED = "cached"; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d3a2964f/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index 2afe0e6..5f98b79 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -94,7 +94,7 @@ public class RangerBasePlugin { serviceName = RangerConfiguration.getInstance().get(propertyPrefix + ".service.name"); - policyEngineOptions.evaluatorType = RangerConfiguration.getInstance().get(propertyPrefix + ".policyengine.option.evaluator.type", RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED); + policyEngineOptions.evaluatorType = RangerConfiguration.getInstance().get(propertyPrefix + ".policyengine.option.evaluator.type", RangerPolicyEvaluator.EVALUATOR_TYPE_AUTO); policyEngineOptions.cacheAuditResults = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", true); policyEngineOptions.disableContextEnrichers = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", false); policyEngineOptions.disableCustomConditions = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", false); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d3a2964f/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java index 7957dbf..37108b1 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java @@ -74,6 +74,11 @@ public interface ServiceStore { ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion) throws Exception; + + Long getServicePolicyVersion(String serviceName); + + ServicePolicies getServicePolicies(String serviceName) throws Exception; + void setPopulateExistingBaseFields(Boolean populateExistingBaseFields); Boolean getPopulateExistingBaseFields(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d3a2964f/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java index 751c3b3..f040bd1 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java @@ -758,6 +758,10 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { return ret; } + @Override + public ServicePolicies getServicePolicies(String serviceName) throws Exception { + return getServicePoliciesIfUpdated(serviceName, -1L); + } private void handleServiceRename(RangerService service, String oldName) throws Exception { List<RangerPolicy> policies = getAllPolicies(); @@ -766,7 +770,6 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { for(RangerPolicy policy : policies) { if(StringUtils.equalsIgnoreCase(policy.getService(), oldName)) { policy.setService(service.getName()); - preUpdate(policy); saveToFile(policy, service.getId(), true); @@ -954,4 +957,16 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore { public Boolean getPopulateExistingBaseFields() { return populateExistingBaseFields; } + + @Override + public Long getServicePolicyVersion(String serviceName) { + RangerService service = null; + try { + service = getServiceByName(serviceName); + } catch (Exception exception) { + LOG.error("Failed to get service object for service:" + serviceName); + } + return service != null ? service.getPolicyVersion() : null; + } + } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d3a2964f/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java index 6c4804d..05c0c76 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java @@ -596,6 +596,11 @@ public class ServiceRESTStore implements ServiceStore { return ret; } + @Override + public ServicePolicies getServicePolicies(String serviceName) throws Exception { + return getServicePoliciesIfUpdated(serviceName, -1L); + } + private WebResource createWebResource(String url) { return createWebResource(url, null); } @@ -629,4 +634,15 @@ public class ServiceRESTStore implements ServiceStore { public Boolean getPopulateExistingBaseFields() { return populateExistingBaseFields; } + + @Override + public Long getServicePolicyVersion(String serviceName) { + RangerService service = null; + try { + service = getServiceByName(serviceName); + } catch (Exception exception) { + LOG.error("Failed to get service object for service:" + serviceName); + } + return service != null ? service.getPolicyVersion() : null; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d3a2964f/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 199d041..6774170 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -19,12 +19,7 @@ package org.apache.ranger.biz; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; import javax.annotation.PostConstruct; @@ -39,7 +34,7 @@ import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.PasswordUtils; import org.apache.ranger.common.RESTErrorUtil; import org.apache.ranger.common.RangerCommonEnums; -import org.apache.ranger.common.RangerConstants; +import org.apache.ranger.common.*; import org.apache.ranger.common.RangerFactory; import org.apache.ranger.common.StringUtil; import org.apache.ranger.common.UserSessionBase; @@ -190,7 +185,8 @@ public class ServiceDBStore implements ServiceStore { public static final String CONFIG_KEY_PASSWORD = "password"; private ServicePredicateUtil predicateUtil = null; - + + @Override public void init() throws Exception { if (LOG.isDebugEnabled()) { @@ -1493,7 +1489,7 @@ public class ServiceDBStore implements ServiceStore { bizUtil.createTrxLog(trxLogList); - LOG.info("Policy Deleted Successfully. PolicyName : " +policyName); + LOG.info("Policy Deleted Successfully. PolicyName : " + policyName); } @Override @@ -1642,7 +1638,7 @@ public class ServiceDBStore implements ServiceStore { @Override public ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion) throws Exception { - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("==> ServiceDBStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ")"); } @@ -1650,31 +1646,78 @@ public class ServiceDBStore implements ServiceStore { XXService serviceDbObj = daoMgr.getXXService().findByName(serviceName); + if (serviceDbObj == null) { + throw new Exception("service does not exist. name=" + serviceName); + } + + if (lastKnownVersion == null || serviceDbObj.getPolicyVersion() == null || !lastKnownVersion.equals(serviceDbObj.getPolicyVersion())) { + ret = RangerServicePoliciesCache.getInstance().getServicePolicies(serviceName, this); + } + + if (ret != null && lastKnownVersion != null && lastKnownVersion.equals(ret.getPolicyVersion())) { + // ServicePolicies are not changed + ret = null; + } + + if (LOG.isDebugEnabled()) { + RangerServicePoliciesCache.getInstance().dump(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDBStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size())); + } + + return ret; + } + + @Override + public Long getServicePolicyVersion(String serviceName) { + + XXService serviceDbObj = daoMgr.getXXService().findByName(serviceName); + + return serviceDbObj != null ? serviceDbObj.getPolicyVersion() : null; + } + + @Override + public ServicePolicies getServicePolicies(String serviceName) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDBStore.getServicePolicies(" + serviceName + ")"); + } + + ServicePolicies ret = null; + + XXService serviceDbObj = daoMgr.getXXService().findByName(serviceName); + if(serviceDbObj == null) { throw new Exception("service does not exist. name=" + serviceName); } - if(lastKnownVersion == null || serviceDbObj.getPolicyVersion() == null || !lastKnownVersion.equals(serviceDbObj.getPolicyVersion())) { - RangerServiceDef serviceDef = getServiceDef(serviceDbObj.getType()); + RangerServiceDef serviceDef = getServiceDef(serviceDbObj.getType()); - if(serviceDef == null) { - throw new Exception("service-def does not exist. id=" + serviceDbObj.getType()); - } + if (serviceDef == null) { + throw new Exception("service-def does not exist. id=" + serviceDbObj.getType()); + } + List<RangerPolicy> policies = null; - List<RangerPolicy> policies = getServicePolicies(serviceDbObj); + if (serviceDbObj.getIsenabled()) { - ret = new ServicePolicies(); + policies = getServicePolicies(serviceDbObj); - ret.setServiceId(serviceDbObj.getId()); - ret.setServiceName(serviceDbObj.getName()); - ret.setPolicyVersion(serviceDbObj.getPolicyVersion()); - ret.setPolicyUpdateTime(serviceDbObj.getPolicyUpdateTime()); - ret.setPolicies(policies); - ret.setServiceDef(serviceDef); + } else { + policies = new ArrayList<RangerPolicy>(); } - if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceDBStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size())); + ret = new ServicePolicies(); + + ret.setServiceId(serviceDbObj.getId()); + ret.setServiceName(serviceDbObj.getName()); + ret.setPolicyVersion(serviceDbObj.getPolicyVersion()); + ret.setPolicyUpdateTime(serviceDbObj.getPolicyUpdateTime()); + ret.setPolicies(policies); + ret.setServiceDef(serviceDef); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDBStore.getServicePolicies(" + serviceName + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size())); } return ret; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d3a2964f/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java new file mode 100644 index 0000000..6c8cbff --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.common; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.store.ServiceStore; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.util.ServicePolicies; + +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +public class RangerServicePoliciesCache { + private static final Log LOG = LogFactory.getLog(RangerServicePoliciesCache.class); + + private static volatile RangerServicePoliciesCache sInstance = null; + private final boolean useServicePoliciesCache; + private final int waitTimeInSeconds; + + private final Map<String, ServicePoliciesWrapper> servicePoliciesMap = new HashMap<String, ServicePoliciesWrapper>(); + + public static RangerServicePoliciesCache getInstance() { + if (sInstance == null) { + synchronized (RangerServicePoliciesCache.class) { + if (sInstance == null) { + sInstance = new RangerServicePoliciesCache(); + } + } + } + return sInstance; + } + + private RangerServicePoliciesCache() { + useServicePoliciesCache = RangerConfiguration.getInstance().getBoolean("ranger.admin.policy.download.usecache", true); + waitTimeInSeconds = RangerConfiguration.getInstance().getInt("ranger.admin.policy.download.cache.max.waittime.for.update", 20); + } + + public void dump() { + + if (useServicePoliciesCache) { + Set<String> serviceNames = null; + + synchronized (this) { + serviceNames = servicePoliciesMap.keySet(); + } + + if (CollectionUtils.isNotEmpty(serviceNames)) { + ServicePoliciesWrapper cachedServicePoliciesWrapper = null; + + for (String serviceName : serviceNames) { + cachedServicePoliciesWrapper = servicePoliciesMap.get(serviceName); + if (LOG.isDebugEnabled()) { + LOG.debug("serviceName:" + serviceName + ", Cached-MetaData:" + cachedServicePoliciesWrapper); + } + } + } + } + } + + public ServicePolicies getServicePolicies(String serviceName) { + + ServicePolicies ret = null; + + if (useServicePoliciesCache && StringUtils.isNotBlank(serviceName)) { + ServicePoliciesWrapper cachedServicePoliciesWrapper = null; + synchronized (this) { + cachedServicePoliciesWrapper = servicePoliciesMap.get(serviceName); + } + if (cachedServicePoliciesWrapper != null) { + ret = cachedServicePoliciesWrapper.getServicePolicies(); + } + } + + return ret; + } + + public ServicePolicies getServicePolicies(String serviceName, ServiceStore serviceStore) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerServicePoliciesCache.getServicePolicies(" + serviceName + ")"); + } + + ServicePolicies ret = null; + + if (StringUtils.isNotBlank(serviceName)) { + + if (LOG.isDebugEnabled()) { + LOG.debug("useServicePoliciesCache=" + useServicePoliciesCache); + } + + ServicePolicies servicePolicies = null; + + if (!useServicePoliciesCache) { + if (serviceStore != null) { + try { + servicePolicies = serviceStore.getServicePolicies(serviceName); + } catch (Exception exception) { + LOG.error("getServicePolicies(" + serviceName + "): failed to get latest policies from service-store", exception); + } + } else { + LOG.error("getServicePolicies(" + serviceName + "): failed to get latest policies as service-store is null!"); + } + } else { + ServicePoliciesWrapper servicePoliciesWrapper = null; + + synchronized (this) { + servicePoliciesWrapper = servicePoliciesMap.get(serviceName); + + if (servicePoliciesWrapper == null) { + servicePoliciesWrapper = new ServicePoliciesWrapper(); + servicePoliciesMap.put(serviceName, servicePoliciesWrapper); + } + } + + if (serviceStore != null) { + boolean refreshed = servicePoliciesWrapper.getLatestOrCached(serviceName, serviceStore); + LOG.info("tryRefreshFromStore returned " + refreshed); + } else { + LOG.error("getServicePolicies(" + serviceName + "): failed to get latest policies as service-store is null!"); + } + + servicePolicies = servicePoliciesWrapper.getServicePolicies(); + } + + ret = servicePolicies; + + } else { + LOG.error("getServicePolicies() failed to get policies as serviceName is null or blank!"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerServicePoliciesCache.getServicePolicies(" + serviceName + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size())); + } + + return ret; + } + + private class ServicePoliciesWrapper { + ServicePolicies servicePolicies; + Date updateTime = null; + long longestDbLoadTimeInMs = -1; + + ReentrantLock lock = new ReentrantLock(); + + ServicePoliciesWrapper() { + servicePolicies = null; + } + + ServicePolicies getServicePolicies() { + return servicePolicies; + } + + Date getUpdateTime() { + return updateTime; + } + + long getLongestDbLoadTimeInMs() { + return longestDbLoadTimeInMs; + } + + boolean getLatestOrCached(String serviceName, ServiceStore serviceStore) { + boolean ret = false; + + try { + ret = lock.tryLock(waitTimeInSeconds, TimeUnit.SECONDS); + if (ret) { + getLatest(serviceName, serviceStore); + } + } catch (InterruptedException exception) { + LOG.error("tryRefreshFromStore:lock got interrupted..", exception); + } finally { + if (ret) { + lock.unlock(); + } + } + + return ret; + } + + void getLatest(String serviceName, ServiceStore serviceStore) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServicePoliciesWrapper.getLatest(" + serviceName + ")"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Found ServicePolicies in-cache : " + (servicePolicies != null)); + } + + Long servicePolicyVersionInDb = serviceStore.getServicePolicyVersion(serviceName); + + + if (servicePolicies == null || servicePolicyVersionInDb == null || !servicePolicyVersionInDb.equals(servicePolicies.getPolicyVersion())) { + if (LOG.isDebugEnabled()) { + LOG.debug("loading servicePolicies from db ... cachedServicePoliciesVersion=" + (servicePolicies != null ? servicePolicies.getPolicyVersion() : null) + ", servicePolicyVersionInDb=" + servicePolicyVersionInDb); + } + + ServicePolicies servicePoliciesFromDb = null; + + try { + long startTimeMs = System.currentTimeMillis(); + + servicePoliciesFromDb = serviceStore.getServicePolicies(serviceName); + + long dbLoadTime = System.currentTimeMillis() - startTimeMs; + + if (dbLoadTime > longestDbLoadTimeInMs) { + longestDbLoadTimeInMs = dbLoadTime; + } + updateTime = new Date(); + } catch (Exception exception) { + LOG.error("getServicePolicies(" + serviceName + "): failed to get latest policies from service-store", exception); + } + + if (servicePoliciesFromDb != null) { + if (servicePoliciesFromDb.getPolicyVersion() == null) { + servicePoliciesFromDb.setPolicyVersion(0L); + } + servicePolicies = servicePoliciesFromDb; + pruneUnusedAttributes(); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServicePoliciesWrapper.getLatest(" + serviceName + ")"); + } + } + + private void pruneUnusedAttributes() { + if (servicePolicies != null) { + pruneUnusedPolicyAttributes(servicePolicies.getPolicies()); + } + } + + private void pruneUnusedPolicyAttributes(List<RangerPolicy> policies) { + + // Null out attributes not required by plug-ins + if (CollectionUtils.isNotEmpty(policies)) { + for (RangerPolicy policy : policies) { + policy.setCreatedBy(null); + policy.setCreateTime(null); + policy.setUpdatedBy(null); + policy.setUpdateTime(null); + policy.setGuid(null); + policy.setName(null); + policy.setDescription(null); + policy.setResourceSignature(null); + } + } + } + + StringBuilder toString(StringBuilder sb) { + sb.append("RangerServicePoliciesWrapper={"); + + sb.append("updateTime=").append(updateTime) + .append(", longestDbLoadTimeInMs=").append(longestDbLoadTimeInMs) + .append(", Service-Version:").append(servicePolicies != null ? servicePolicies.getPolicyVersion() : "null") + .append(", Number-Of-Policies:").append(servicePolicies != null ? servicePolicies.getPolicies().size() : 0); + + sb.append("} "); + + return sb; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + } +} + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/d3a2964f/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index f744684..40628bb 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -1906,7 +1906,7 @@ public class ServiceREST { String propertyPrefix = "ranger.admin"; - options.evaluatorType = RangerConfiguration.getInstance().get(propertyPrefix + ".policyengine.option.evaluator.type", RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED); + options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; options.cacheAuditResults = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", false); options.disableContextEnrichers = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true); options.disableCustomConditions = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true);
