Repository: incubator-ranger Updated Branches: refs/heads/master 6ca739b04 -> 4e2e83eef
RANGER-836: Optimize policy download to plugins Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/4e2e83ee Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/4e2e83ee Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/4e2e83ee Branch: refs/heads/master Commit: 4e2e83eef660fae5287d4d1dc7bbea68015445ee Parents: 6ca739b Author: Abhay Kulkarni <[email protected]> Authored: Mon Feb 1 12:07:41 2016 -0800 Committer: Madhan Neethiraj <[email protected]> Committed: Wed Feb 10 17:24:34 2016 -0800 ---------------------------------------------------------------------- .../policyengine/RangerPolicyEngineImpl.java | 24 ++ .../policyengine/RangerPolicyEngineOptions.java | 3 +- .../policyengine/RangerPolicyRepository.java | 47 ++- .../policyevaluator/RangerPolicyEvaluator.java | 2 +- .../ranger/plugin/service/RangerBasePlugin.java | 2 +- .../plugin/store/AbstractServiceStore.java | 11 + .../ranger/plugin/store/ServiceStore.java | 4 + .../apache/ranger/plugin/store/TagStore.java | 4 + .../plugin/store/file/ServiceFileStore.java | 5 +- .../ranger/plugin/store/file/TagFileStore.java | 119 +++++--- .../plugin/store/rest/ServiceRESTStore.java | 5 + .../apache/ranger/plugin/util/ServiceTags.java | 1 - .../org/apache/ranger/biz/ServiceDBStore.java | 141 +++++---- .../java/org/apache/ranger/biz/TagDBStore.java | 121 +++++--- .../common/RangerServicePoliciesCache.java | 301 ++++++++++++++++++ .../ranger/common/RangerServiceTagsCache.java | 306 +++++++++++++++++++ .../org/apache/ranger/rest/ServiceREST.java | 3 +- 17 files changed, 949 insertions(+), 150 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index 6222f47..022f5a7 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@ -23,6 +23,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.contextenricher.RangerContextEnricher; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerTag; @@ -43,6 +44,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { private static final Log PERF_POLICYENGINE_AUDIT_LOG = RangerPerfTracer.getPerfLogger("policyengine.audit"); private static final Log PERF_CONTEXTENRICHER_REQUEST_LOG = RangerPerfTracer.getPerfLogger("contextenricher.request"); + private static final int MAX_POLICIES_FOR_CACHE_TYPE_EVALUATOR = 500; + private final RangerPolicyRepository policyRepository; private final RangerPolicyRepository tagPolicyRepository; @@ -63,6 +66,27 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { options = new RangerPolicyEngineOptions(); } + if(StringUtils.isBlank(options.evaluatorType) || StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_AUTO)) { + + String serviceType = servicePolicies.getServiceDef().getName(); + String propertyName = "ranger.plugin." + serviceType + ".policyengine.evaluator.auto.maximum.policycount.for.cache.type"; + + int thresholdForUsingOptimizedEvaluator = RangerConfiguration.getInstance().getInt(propertyName, MAX_POLICIES_FOR_CACHE_TYPE_EVALUATOR); + + int servicePoliciesCount = servicePolicies.getPolicies().size() + (servicePolicies.getTagPolicies() != null ? servicePolicies.getTagPolicies().getPolicies().size() : 0); + + if (servicePoliciesCount > thresholdForUsingOptimizedEvaluator) { + options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; + } else { + options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED; + } + } else if (StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED)) { + options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED; + } else { + // All other cases + options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; + } + policyRepository = new RangerPolicyRepository(appId, servicePolicies, options); ServicePolicies.TagPolicies tagPolicies = servicePolicies.getTagPolicies(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java index dcd1b3c..805f5a5 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java @@ -23,9 +23,10 @@ import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; public class RangerPolicyEngineOptions { - public String evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED; + public String evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_AUTO; public boolean cacheAuditResults = true; public boolean disableContextEnrichers = false; public boolean disableCustomConditions = false; public boolean disableTagPolicyEvaluation = true; + public boolean evaluateDelegateAdminOnly = false; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index 641320f..e79b5cd 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -270,11 +270,48 @@ public class RangerPolicyRepository { return policyItems; } + public static boolean isDelegateAdminPolicy(RangerPolicy policy) { + boolean ret = false; + + ret = hasDelegateAdminItems(policy.getPolicyItems()) + || hasDelegateAdminItems(policy.getDenyPolicyItems()) + || hasDelegateAdminItems(policy.getAllowExceptions()) + || hasDelegateAdminItems(policy.getDenyExceptions()); + + return ret; + } + + private static boolean hasDelegateAdminItems(List<RangerPolicy.RangerPolicyItem> items) { + boolean ret = false; + + if (CollectionUtils.isNotEmpty(items)) { + for (RangerPolicy.RangerPolicyItem item : items) { + if(item.getDelegateAdmin()) { + ret = true; + + break; + } + } + } + return ret; + } + + private static boolean skipBuildingPolicyEvaluator(RangerPolicy policy, RangerPolicyEngineOptions options) { + boolean ret = false; + if (!policy.getIsEnabled()) { + ret = true; + } else if (options.evaluateDelegateAdminOnly && !isDelegateAdminPolicy(policy)) { + ret = true; + } + return ret; + } + private void init(RangerPolicyEngineOptions options) { List<RangerPolicyEvaluator> policyEvaluators = new ArrayList<RangerPolicyEvaluator>(); + for (RangerPolicy policy : policies) { - if (!policy.getIsEnabled()) { + if (skipBuildingPolicyEvaluator(policy, options)) { continue; } @@ -367,12 +404,10 @@ public class RangerPolicyRepository { RangerPolicyEvaluator ret; - if(StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_DEFAULT)) { - ret = new RangerOptimizedPolicyEvaluator(); - } else if(StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED)) { - ret = new RangerOptimizedPolicyEvaluator(); - } else { + if(StringUtils.equalsIgnoreCase(options.evaluatorType, RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED)) { ret = new RangerCachedPolicyEvaluator(); + } else { + ret = new RangerOptimizedPolicyEvaluator(); } ret.init(policy, serviceDef, options); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java index e5f34a2..9cb90f4 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java @@ -32,7 +32,7 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResource; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; public interface RangerPolicyEvaluator extends Comparable<RangerPolicyEvaluator> { - public static final String EVALUATOR_TYPE_DEFAULT = "default"; + public static final String EVALUATOR_TYPE_AUTO = "auto"; public static final String EVALUATOR_TYPE_OPTIMIZED = "optimized"; public static final String EVALUATOR_TYPE_CACHED = "cached"; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index 9d9987b..24749bb 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -95,7 +95,7 @@ public class RangerBasePlugin { serviceName = RangerConfiguration.getInstance().get(propertyPrefix + ".service.name"); - policyEngineOptions.evaluatorType = RangerConfiguration.getInstance().get(propertyPrefix + ".policyengine.option.evaluator.type", RangerPolicyEvaluator.EVALUATOR_TYPE_CACHED); + policyEngineOptions.evaluatorType = RangerConfiguration.getInstance().get(propertyPrefix + ".policyengine.option.evaluator.type", RangerPolicyEvaluator.EVALUATOR_TYPE_AUTO); policyEngineOptions.cacheAuditResults = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", true); policyEngineOptions.disableContextEnrichers = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", false); policyEngineOptions.disableCustomConditions = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", false); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java index e9aae60..0dead8a 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java @@ -137,6 +137,17 @@ public abstract class AbstractServiceStore implements ServiceStore { return ret == null ? null : ret.getName(); } + @Override + public Long getServicePolicyVersion(String serviceName) { + RangerService service = null; + try { + service = getServiceByName(serviceName); + } catch (Exception exception) { + LOG.error("Failed to get service object for service:" + serviceName); + } + return service != null ? service.getPolicyVersion() : null; + } + protected void preCreate(RangerBaseModelObject obj) throws Exception { obj.setId(0L); if(obj.getGuid() == null) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java index 8013262..febe640 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java @@ -89,6 +89,10 @@ public interface ServiceStore { ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion) throws Exception; + Long getServicePolicyVersion(String serviceName); + + ServicePolicies getServicePolicies(String serviceName) throws Exception; + RangerPolicy getPolicyFromEventTime(String eventTimeStr, Long policyId); RangerPolicy getPolicyForVersionNumber(Long policyId, Integer versionNo); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java index 104459d..69bd628 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java @@ -124,5 +124,9 @@ public interface TagStore { ServiceTags getServiceTagsIfUpdated(String serviceName, Long lastKnownVersion) throws Exception; + ServiceTags getServiceTags(String serviceName) throws Exception; + + Long getTagVersion(String serviceName); + void deleteAllTagObjectsForService(String serviceName, boolean isResourePrivateTag) throws Exception; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java index 8509f63..a68a6ed 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java @@ -779,6 +779,10 @@ public class ServiceFileStore extends AbstractServiceStore { return ret; } + @Override + public ServicePolicies getServicePolicies(String serviceName) throws Exception { + return getServicePoliciesIfUpdated(serviceName, -1L); + } private void handleServiceRename(RangerService service, String oldName) throws Exception { List<RangerPolicy> policies = getAllPolicies(); @@ -787,7 +791,6 @@ public class ServiceFileStore extends AbstractServiceStore { for(RangerPolicy policy : policies) { if(StringUtils.equalsIgnoreCase(policy.getService(), oldName)) { policy.setService(service.getName()); - preUpdate(policy); fileStoreUtil.saveToFile(policy, FILE_PREFIX_POLICY, service.getId(), true); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java index 14b0d73..6f15fd1 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java @@ -1076,9 +1076,7 @@ public class TagFileStore extends AbstractTagStore { LOG.debug("==> TagFileStore.getServiceTagsIfUpdated(" + serviceName + ", " + lastKnownVersion + ")"); } - ServiceTags ret = new ServiceTags(); - ret.setOp(ServiceTags.OP_ADD_OR_UPDATE); - ret.setTagModel(ServiceTags.TAGMODEL_SHARED); + ServiceTags ret = null; boolean tagsChanged = true; @@ -1086,7 +1084,6 @@ public class TagFileStore extends AbstractTagStore { try { service = svcStore.getServiceByName(serviceName); - ret.setServiceName(serviceName); } catch (Exception exception) { LOG.error("Cannot find service for serviceName=" + serviceName); tagsChanged = false; @@ -1094,55 +1091,21 @@ public class TagFileStore extends AbstractTagStore { if (lastKnownVersion != null && service != null && service.getTagVersion() != null - && lastKnownVersion.compareTo(service.getTagVersion()) >= 0 ) { + && lastKnownVersion.equals(service.getTagVersion())) { tagsChanged = false; } if (tagsChanged) { - SearchFilter filter = new SearchFilter(); - - filter.setParam(SearchFilter.TAG_RESOURCE_SERVICE_NAME, serviceName); - - List<RangerServiceResource> serviceResources = getServiceResources(filter); - List<RangerServiceResource> filteredServiceResources = new ArrayList<RangerServiceResource>(); - - Map<Long, RangerTag> tagsMap = new HashMap<Long, RangerTag>(); - Map<Long, List<Long>> resourceToTagIdsMap = new HashMap<Long, List<Long>>(); - - for (RangerServiceResource serviceResource : serviceResources) { - List<RangerTag> tagList = getTagsForServiceResourceObject(serviceResource); - - if (CollectionUtils.isNotEmpty(tagList)) { - List<Long> tagIdList = new ArrayList<Long>(); - for (RangerTag tag : tagList) { - tagsMap.put(tag.getId(), tag); - tagIdList.add(tag.getId()); - } - resourceToTagIdsMap.put(serviceResource.getId(), tagIdList); - filteredServiceResources.add(serviceResource); - } - } - - ret.setServiceResources(filteredServiceResources); - ret.setResourceToTagIds(resourceToTagIdsMap); - ret.setTags(tagsMap); - - if (service != null && service.getTagVersion() != null) { - ret.setTagVersion(service.getTagVersion()); - } - if (service != null && service.getTagUpdateTime() != null) { - ret.setTagUpdateTime(service.getTagUpdateTime()); - } if (LOG.isDebugEnabled()) { LOG.debug("Changes to tagVersion detected, tagVersion in service=" + (service == null ? null : service.getTagVersion()) + ", Plugin-provided lastKnownVersion=" + lastKnownVersion); } + ret = getServiceTags(serviceName); } else { if (LOG.isDebugEnabled()) { LOG.debug("No changes to tagVersion detected, tagVersion in service=" + (service == null ? null : service.getTagVersion()) - + ", Plugin-provided lastKnownVersion=" + lastKnownVersion); + + ", Plugin-provided lastKnownVersion=" + lastKnownVersion); } - ret.setTagVersion(lastKnownVersion); } if (LOG.isDebugEnabled()) { @@ -1150,7 +1113,81 @@ public class TagFileStore extends AbstractTagStore { } return ret; + } + + @Override + public ServiceTags getServiceTags(String serviceName) throws Exception { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagFileStore.getServiceTags(" + serviceName + ")"); + } + + ServiceTags ret = new ServiceTags(); + + RangerService service = null; + + try { + service = svcStore.getServiceByName(serviceName); + } catch (Exception exception) { + LOG.error("Cannot find service for serviceName=" + serviceName); + } + + SearchFilter filter = new SearchFilter(); + + filter.setParam(SearchFilter.TAG_RESOURCE_SERVICE_NAME, serviceName); + + List<RangerServiceResource> serviceResources = getServiceResources(filter); + List<RangerServiceResource> filteredServiceResources = new ArrayList<RangerServiceResource>(); + + Map<Long, RangerTag> tagsMap = new HashMap<Long, RangerTag>(); + Map<Long, List<Long>> resourceToTagIdsMap = new HashMap<Long, List<Long>>(); + + for (RangerServiceResource serviceResource : serviceResources) { + List<RangerTag> tagList = getTagsForServiceResourceObject(serviceResource); + + if (CollectionUtils.isNotEmpty(tagList)) { + List<Long> tagIdList = new ArrayList<Long>(); + for (RangerTag tag : tagList) { + tagsMap.put(tag.getId(), tag); + tagIdList.add(tag.getId()); + } + resourceToTagIdsMap.put(serviceResource.getId(), tagIdList); + filteredServiceResources.add(serviceResource); + } + } + + ret.setServiceName(serviceName); + ret.setServiceResources(filteredServiceResources); + ret.setResourceToTagIds(resourceToTagIdsMap); + ret.setTags(tagsMap); + + if (service != null && service.getTagVersion() != null) { + ret.setTagVersion(service.getTagVersion()); + } + if (service != null && service.getTagUpdateTime() != null) { + ret.setTagUpdateTime(service.getTagUpdateTime()); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagFileStore.getServiceTags(" + serviceName + "): " + ret); + } + + return ret; + + } + + @Override + public Long getTagVersion(String serviceName) { + + RangerService service = null; + + try { + service = svcStore.getServiceByName(serviceName); + } catch (Exception exception) { + LOG.error("Cannot find service for serviceName=" + serviceName); + } + return service != null ? service.getTagVersion() : null; } private List<RangerTag> getTagsForServiceResourceObject(RangerServiceResource serviceResource) throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java index 5886cf5..2717551 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/rest/ServiceRESTStore.java @@ -600,6 +600,11 @@ public class ServiceRESTStore extends AbstractServiceStore { return ret; } + @Override + public ServicePolicies getServicePolicies(String serviceName) throws Exception { + return getServicePoliciesIfUpdated(serviceName, -1L); + } + private WebResource createWebResource(String url) { return createWebResource(url, null); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java index c3fb4cf..738a947 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java @@ -202,5 +202,4 @@ public class ServiceTags implements java.io.Serializable { return sb; } - } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 38091bd..68e64c3 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -19,12 +19,7 @@ package org.apache.ranger.biz; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; import javax.annotation.PostConstruct; @@ -34,17 +29,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; -import org.apache.ranger.common.AppConstants; -import org.apache.ranger.common.ContextUtil; -import org.apache.ranger.common.DateUtil; -import org.apache.ranger.common.MessageEnums; -import org.apache.ranger.common.PasswordUtils; -import org.apache.ranger.common.RESTErrorUtil; -import org.apache.ranger.common.RangerCommonEnums; -import org.apache.ranger.common.RangerConstants; -import org.apache.ranger.common.RangerFactory; -import org.apache.ranger.common.StringUtil; -import org.apache.ranger.common.UserSessionBase; +import org.apache.ranger.common.*; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.db.XXAccessTypeDefDao; import org.apache.ranger.db.XXAccessTypeDefGrantsDao; @@ -192,7 +177,8 @@ public class ServiceDBStore extends AbstractServiceStore { public static final String CONFIG_KEY_PASSWORD = "password"; private ServicePredicateUtil predicateUtil = null; - + + @Override public void init() throws Exception { if (LOG.isDebugEnabled()) { @@ -1576,7 +1562,7 @@ public class ServiceDBStore extends AbstractServiceStore { bizUtil.createTrxLog(trxLogList); - LOG.info("Policy Deleted Successfully. PolicyName : " +policyName); + LOG.info("Policy Deleted Successfully. PolicyName : " + policyName); } @Override @@ -1736,7 +1722,7 @@ public class ServiceDBStore extends AbstractServiceStore { @Override public ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion) throws Exception { - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("==> ServiceDBStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + ")"); } @@ -1744,60 +1730,100 @@ public class ServiceDBStore extends AbstractServiceStore { XXService serviceDbObj = daoMgr.getXXService().findByName(serviceName); - if(serviceDbObj == null) { + if (serviceDbObj == null) { throw new Exception("service does not exist. name=" + serviceName); } - if(lastKnownVersion == null || serviceDbObj.getPolicyVersion() == null || !lastKnownVersion.equals(serviceDbObj.getPolicyVersion())) { - RangerServiceDef serviceDef = getServiceDef(serviceDbObj.getType()); + if (lastKnownVersion == null || serviceDbObj.getPolicyVersion() == null || !lastKnownVersion.equals(serviceDbObj.getPolicyVersion())) { + ret = RangerServicePoliciesCache.getInstance().getServicePolicies(serviceName, this); + } - if(serviceDef == null) { - throw new Exception("service-def does not exist. id=" + serviceDbObj.getType()); - } + if (ret != null && lastKnownVersion != null && lastKnownVersion.equals(ret.getPolicyVersion())) { + // ServicePolicies are not changed + ret = null; + } - List<RangerPolicy> policies = null; - ServicePolicies.TagPolicies tagPolicies = null; + if (LOG.isDebugEnabled()) { + RangerServicePoliciesCache.getInstance().dump(); + } - if (serviceDbObj.getIsenabled()) { - if (serviceDbObj.getTagService() != null) { - XXService tagServiceDbObj = daoMgr.getXXService().getById(serviceDbObj.getTagService()); + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDBStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size())); + } + + return ret; + } - if (tagServiceDbObj != null && tagServiceDbObj.getIsenabled()) { - RangerServiceDef tagServiceDef = getServiceDef(tagServiceDbObj.getType()); + @Override + public Long getServicePolicyVersion(String serviceName) { - if (tagServiceDef == null) { - throw new Exception("service-def does not exist. id=" + tagServiceDbObj.getType()); - } + XXService serviceDbObj = daoMgr.getXXService().findByName(serviceName); + + return serviceDbObj != null ? serviceDbObj.getPolicyVersion() : null; + } - tagPolicies = new ServicePolicies.TagPolicies(); + @Override + public ServicePolicies getServicePolicies(String serviceName) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceDBStore.getServicePolicies(" + serviceName + ")"); + } - tagPolicies.setServiceId(tagServiceDbObj.getId()); - tagPolicies.setServiceName(tagServiceDbObj.getName()); - tagPolicies.setPolicyVersion(tagServiceDbObj.getPolicyVersion()); - tagPolicies.setPolicyUpdateTime(tagServiceDbObj.getPolicyUpdateTime()); - tagPolicies.setPolicies(getServicePolicies(tagServiceDbObj)); - tagPolicies.setServiceDef(tagServiceDef); + ServicePolicies ret = null; + + XXService serviceDbObj = daoMgr.getXXService().findByName(serviceName); + + if(serviceDbObj == null) { + throw new Exception("service does not exist. name=" + serviceName); + } + + RangerServiceDef serviceDef = getServiceDef(serviceDbObj.getType()); + + if (serviceDef == null) { + throw new Exception("service-def does not exist. id=" + serviceDbObj.getType()); + } + List<RangerPolicy> policies = null; + ServicePolicies.TagPolicies tagPolicies = null; + + if (serviceDbObj.getIsenabled()) { + if (serviceDbObj.getTagService() != null) { + XXService tagServiceDbObj = daoMgr.getXXService().getById(serviceDbObj.getTagService()); + + if (tagServiceDbObj != null && tagServiceDbObj.getIsenabled()) { + RangerServiceDef tagServiceDef = getServiceDef(tagServiceDbObj.getType()); + + if (tagServiceDef == null) { + throw new Exception("service-def does not exist. id=" + tagServiceDbObj.getType()); } - } - policies = getServicePolicies(serviceDbObj); - } else { - policies = new ArrayList<RangerPolicy>(); + tagPolicies = new ServicePolicies.TagPolicies(); + + tagPolicies.setServiceId(tagServiceDbObj.getId()); + tagPolicies.setServiceName(tagServiceDbObj.getName()); + tagPolicies.setPolicyVersion(tagServiceDbObj.getPolicyVersion()); + tagPolicies.setPolicyUpdateTime(tagServiceDbObj.getPolicyUpdateTime()); + tagPolicies.setPolicies(getServicePolicies(tagServiceDbObj)); + tagPolicies.setServiceDef(tagServiceDef); + } } - ret = new ServicePolicies(); + policies = getServicePolicies(serviceDbObj); - ret.setServiceId(serviceDbObj.getId()); - ret.setServiceName(serviceDbObj.getName()); - ret.setPolicyVersion(serviceDbObj.getPolicyVersion()); - ret.setPolicyUpdateTime(serviceDbObj.getPolicyUpdateTime()); - ret.setPolicies(policies); - ret.setServiceDef(serviceDef); - ret.setTagPolicies(tagPolicies); + } else { + policies = new ArrayList<RangerPolicy>(); } - if(LOG.isDebugEnabled()) { - LOG.debug("<== ServiceDBStore.getServicePoliciesIfUpdated(" + serviceName + ", " + lastKnownVersion + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size())); + ret = new ServicePolicies(); + + ret.setServiceId(serviceDbObj.getId()); + ret.setServiceName(serviceDbObj.getName()); + ret.setPolicyVersion(serviceDbObj.getPolicyVersion()); + ret.setPolicyUpdateTime(serviceDbObj.getPolicyUpdateTime()); + ret.setPolicies(policies); + ret.setServiceDef(serviceDef); + ret.setTagPolicies(tagPolicies); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceDBStore.getServicePolicies(" + serviceName + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size())); } return ret; @@ -2346,4 +2372,5 @@ public class ServiceDBStore extends AbstractServiceStore { return ret; } + } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java index f89a434..bf56afa 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java @@ -32,6 +32,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.ranger.common.GUIDUtil; import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.RESTErrorUtil; +import org.apache.ranger.common.RangerServiceTagsCache; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.entity.XXDBBase; import org.apache.ranger.entity.XXResourceDef; @@ -827,7 +828,7 @@ public class TagDBStore extends AbstractTagStore { @Override public ServiceTags getServiceTagsIfUpdated(String serviceName, Long lastKnownVersion) throws Exception { - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("==> TagDBStore.getServiceTagsIfUpdated(" + serviceName + ", " + lastKnownVersion + ")"); } @@ -835,65 +836,105 @@ public class TagDBStore extends AbstractTagStore { XXService xxService = daoManager.getXXService().findByName(serviceName); - if(xxService == null) { + if (xxService == null) { throw new Exception("service does not exist. name=" + serviceName); } - if(lastKnownVersion == null || xxService.getTagVersion() == null || !lastKnownVersion.equals(xxService.getTagVersion())) { - RangerServiceDef serviceDef = svcStore.getServiceDef(xxService.getType()); + if (lastKnownVersion == null || xxService.getTagVersion() == null || !lastKnownVersion.equals(xxService.getTagVersion())) { + ret = RangerServiceTagsCache.getInstance().getServiceTags(serviceName, this); + } - if(serviceDef == null) { - throw new Exception("service-def does not exist. id=" + xxService.getType()); - } + if (ret != null && lastKnownVersion != null && lastKnownVersion.equals(ret.getTagVersion())) { + // ServiceTags are not changed + ret = null; + } - RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, xxService); + if (LOG.isDebugEnabled()) { + RangerServiceTagsCache.getInstance().dump(); + } - Map<Long, RangerTagDef> tagDefMap = tagDBRetriever.getTagDefs(); - Map<Long, RangerTag> tagMap = tagDBRetriever.getTags(); - List<RangerServiceResource> resources = tagDBRetriever.getServiceResources(); - List<RangerTagResourceMap> tagResourceMaps = tagDBRetriever.getTagResourceMaps(); + if (LOG.isDebugEnabled()) { + LOG.debug("<== TagDBStore.getServiceTagsIfUpdated(" + serviceName + ", " + lastKnownVersion + "): count=" + ((ret == null || ret.getTags() == null) ? 0 : ret.getTags().size())); + } - Map<Long, List<Long>> resourceToTagIds = new HashMap<Long, List<Long>>(); + return ret; + } - if(CollectionUtils.isNotEmpty(tagResourceMaps)) { - Long resourceId = null; - List<Long> tagIds = null; + @Override + public Long getTagVersion(String serviceName) { + + XXService serviceDbObj = daoManager.getXXService().findByName(serviceName); + + return serviceDbObj != null ? serviceDbObj.getTagVersion() : null; + } + + @Override + public ServiceTags getServiceTags(String serviceName) throws Exception { - for(RangerTagResourceMap tagResourceMap : tagResourceMaps) { - if(! tagResourceMap.getResourceId().equals(resourceId)) { - if(resourceId != null) { - resourceToTagIds.put(resourceId, tagIds); - } + if (LOG.isDebugEnabled()) { + LOG.debug("==> TagDBStore.getServiceTags(" + serviceName + ")"); + } + + ServiceTags ret = null; - resourceId = tagResourceMap.getResourceId(); - tagIds = new ArrayList<Long>(); + XXService xxService = daoManager.getXXService().findByName(serviceName); + + if (xxService == null) { + throw new Exception("service does not exist. name=" + serviceName); + } + + RangerServiceDef serviceDef = svcStore.getServiceDef(xxService.getType()); + + if (serviceDef == null) { + throw new Exception("service-def does not exist. id=" + xxService.getType()); + } + + RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, xxService); + + Map<Long, RangerTagDef> tagDefMap = tagDBRetriever.getTagDefs(); + Map<Long, RangerTag> tagMap = tagDBRetriever.getTags(); + List<RangerServiceResource> resources = tagDBRetriever.getServiceResources(); + List<RangerTagResourceMap> tagResourceMaps = tagDBRetriever.getTagResourceMaps(); + + Map<Long, List<Long>> resourceToTagIds = new HashMap<Long, List<Long>>(); + + if (CollectionUtils.isNotEmpty(tagResourceMaps)) { + Long resourceId = null; + List<Long> tagIds = null; + + for (RangerTagResourceMap tagResourceMap : tagResourceMaps) { + if (!tagResourceMap.getResourceId().equals(resourceId)) { + if (resourceId != null) { + resourceToTagIds.put(resourceId, tagIds); } - tagIds.add(tagResourceMap.getTagId()); - } - - if(resourceId != null) { - resourceToTagIds.put(resourceId, tagIds); + resourceId = tagResourceMap.getResourceId(); + tagIds = new ArrayList<Long>(); } + + tagIds.add(tagResourceMap.getTagId()); } - ret = new ServiceTags(); - ret.setOp(ServiceTags.OP_ADD_OR_UPDATE); - ret.setTagModel(ServiceTags.TAGMODEL_SHARED); - ret.setServiceName(xxService.getName()); - ret.setTagVersion(xxService.getTagVersion()); - ret.setTagUpdateTime(xxService.getTagUpdateTime()); - ret.setTagDefinitions(tagDefMap); - ret.setTags(tagMap); - ret.setServiceResources(resources); - ret.setResourceToTagIds(resourceToTagIds); + if (resourceId != null) { + resourceToTagIds.put(resourceId, tagIds); + } } + ret = new ServiceTags(); + + ret.setServiceName(xxService.getName()); + ret.setTagVersion(xxService.getTagVersion()); + ret.setTagUpdateTime(xxService.getTagUpdateTime()); + ret.setTagDefinitions(tagDefMap); + ret.setTags(tagMap); + ret.setServiceResources(resources); + ret.setResourceToTagIds(resourceToTagIds); + if(LOG.isDebugEnabled()) { - LOG.debug("<== TagDBStore.getServiceTagsIfUpdated(" + serviceName + ", " + lastKnownVersion + ")"); + LOG.debug("<== TagDBStore.getServiceTags(" + serviceName + ")"); } - return ret; + } private List<XXTagAttributeDef> createTagAttributeDefs(Long tagDefId, List<RangerTagAttributeDef> tagAttrDefList) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java new file mode 100644 index 0000000..f4afa3e --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.common; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.plugin.model.RangerPolicy; +import org.apache.ranger.plugin.store.ServiceStore; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.util.ServicePolicies; + +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +public class RangerServicePoliciesCache { + private static final Log LOG = LogFactory.getLog(RangerServicePoliciesCache.class); + + private static volatile RangerServicePoliciesCache sInstance = null; + private final boolean useServicePoliciesCache; + private final int waitTimeInSeconds; + + private final Map<String, ServicePoliciesWrapper> servicePoliciesMap = new HashMap<String, ServicePoliciesWrapper>(); + + public static RangerServicePoliciesCache getInstance() { + if (sInstance == null) { + synchronized (RangerServicePoliciesCache.class) { + if (sInstance == null) { + sInstance = new RangerServicePoliciesCache(); + } + } + } + return sInstance; + } + + private RangerServicePoliciesCache() { + useServicePoliciesCache = RangerConfiguration.getInstance().getBoolean("ranger.admin.policy.download.usecache", true); + waitTimeInSeconds = RangerConfiguration.getInstance().getInt("ranger.admin.policy.download.cache.max.waittime.for.update", 20); + } + + public void dump() { + + if (useServicePoliciesCache) { + Set<String> serviceNames = null; + + synchronized (this) { + serviceNames = servicePoliciesMap.keySet(); + } + + if (CollectionUtils.isNotEmpty(serviceNames)) { + ServicePoliciesWrapper cachedServicePoliciesWrapper = null; + + for (String serviceName : serviceNames) { + cachedServicePoliciesWrapper = servicePoliciesMap.get(serviceName); + if (LOG.isDebugEnabled()) { + LOG.debug("serviceName:" + serviceName + ", Cached-MetaData:" + cachedServicePoliciesWrapper); + } + } + } + } + } + + public ServicePolicies getServicePolicies(String serviceName) { + + ServicePolicies ret = null; + + if (useServicePoliciesCache && StringUtils.isNotBlank(serviceName)) { + ServicePoliciesWrapper cachedServicePoliciesWrapper = null; + synchronized (this) { + cachedServicePoliciesWrapper = servicePoliciesMap.get(serviceName); + } + if (cachedServicePoliciesWrapper != null) { + ret = cachedServicePoliciesWrapper.getServicePolicies(); + } + } + + return ret; + } + + public ServicePolicies getServicePolicies(String serviceName, ServiceStore serviceStore) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerServicePoliciesCache.getServicePolicies(" + serviceName + ")"); + } + + ServicePolicies ret = null; + + if (StringUtils.isNotBlank(serviceName)) { + + if (LOG.isDebugEnabled()) { + LOG.debug("useServicePoliciesCache=" + useServicePoliciesCache); + } + + ServicePolicies servicePolicies = null; + + if (!useServicePoliciesCache) { + if (serviceStore != null) { + try { + servicePolicies = serviceStore.getServicePolicies(serviceName); + } catch (Exception exception) { + LOG.error("getServicePolicies(" + serviceName + "): failed to get latest policies from service-store", exception); + } + } else { + LOG.error("getServicePolicies(" + serviceName + "): failed to get latest policies as service-store is null!"); + } + } else { + ServicePoliciesWrapper servicePoliciesWrapper = null; + + synchronized (this) { + servicePoliciesWrapper = servicePoliciesMap.get(serviceName); + + if (servicePoliciesWrapper == null) { + servicePoliciesWrapper = new ServicePoliciesWrapper(); + servicePoliciesMap.put(serviceName, servicePoliciesWrapper); + } + } + + if (serviceStore != null) { + boolean refreshed = servicePoliciesWrapper.getLatestOrCached(serviceName, serviceStore); + LOG.info("tryRefreshFromStore returned " + refreshed); + } else { + LOG.error("getServicePolicies(" + serviceName + "): failed to get latest policies as service-store is null!"); + } + + servicePolicies = servicePoliciesWrapper.getServicePolicies(); + } + + ret = servicePolicies; + + } else { + LOG.error("getServicePolicies() failed to get policies as serviceName is null or blank!"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerServicePoliciesCache.getServicePolicies(" + serviceName + "): count=" + ((ret == null || ret.getPolicies() == null) ? 0 : ret.getPolicies().size())); + } + + return ret; + } + + private class ServicePoliciesWrapper { + ServicePolicies servicePolicies; + Date updateTime = null; + long longestDbLoadTimeInMs = -1; + + ReentrantLock lock = new ReentrantLock(); + + ServicePoliciesWrapper() { + servicePolicies = null; + } + + ServicePolicies getServicePolicies() { + return servicePolicies; + } + + Date getUpdateTime() { + return updateTime; + } + + long getLongestDbLoadTimeInMs() { + return longestDbLoadTimeInMs; + } + + boolean getLatestOrCached(String serviceName, ServiceStore serviceStore) { + boolean ret = false; + + try { + ret = lock.tryLock(waitTimeInSeconds, TimeUnit.SECONDS); + if (ret) { + getLatest(serviceName, serviceStore); + } + } catch (InterruptedException exception) { + LOG.error("tryRefreshFromStore:lock got interrupted..", exception); + } finally { + if (ret) { + lock.unlock(); + } + } + + return ret; + } + + void getLatest(String serviceName, ServiceStore serviceStore) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServicePoliciesWrapper.getLatest(" + serviceName + ")"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Found ServicePolicies in-cache : " + (servicePolicies != null)); + } + + Long servicePolicyVersionInDb = serviceStore.getServicePolicyVersion(serviceName); + + + if (servicePolicies == null || servicePolicyVersionInDb == null || !servicePolicyVersionInDb.equals(servicePolicies.getPolicyVersion())) { + if (LOG.isDebugEnabled()) { + LOG.debug("loading servicePolicies from db ... cachedServicePoliciesVersion=" + (servicePolicies != null ? servicePolicies.getPolicyVersion() : null) + ", servicePolicyVersionInDb=" + servicePolicyVersionInDb); + } + + ServicePolicies servicePoliciesFromDb = null; + + try { + long startTimeMs = System.currentTimeMillis(); + + servicePoliciesFromDb = serviceStore.getServicePolicies(serviceName); + + long dbLoadTime = System.currentTimeMillis() - startTimeMs; + + if (dbLoadTime > longestDbLoadTimeInMs) { + longestDbLoadTimeInMs = dbLoadTime; + } + updateTime = new Date(); + } catch (Exception exception) { + LOG.error("getServicePolicies(" + serviceName + "): failed to get latest policies from service-store", exception); + } + + if (servicePoliciesFromDb != null) { + if (servicePoliciesFromDb.getPolicyVersion() == null) { + servicePoliciesFromDb.setPolicyVersion(0L); + } + servicePolicies = servicePoliciesFromDb; + pruneUnusedAttributes(); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServicePoliciesWrapper.getLatest(" + serviceName + ")"); + } + } + + private void pruneUnusedAttributes() { + if (servicePolicies != null) { + pruneUnusedPolicyAttributes(servicePolicies.getPolicies()); + if (servicePolicies.getTagPolicies() != null) { + pruneUnusedPolicyAttributes(servicePolicies.getTagPolicies().getPolicies()); + } + } + } + + private void pruneUnusedPolicyAttributes(List<RangerPolicy> policies) { + + // Null out attributes not required by plug-ins + if (CollectionUtils.isNotEmpty(policies)) { + for (RangerPolicy policy : policies) { + policy.setCreatedBy(null); + policy.setCreateTime(null); + policy.setUpdatedBy(null); + policy.setUpdateTime(null); + policy.setGuid(null); + policy.setName(null); + policy.setDescription(null); + policy.setResourceSignature(null); + } + } + } + + StringBuilder toString(StringBuilder sb) { + sb.append("RangerServicePoliciesWrapper={"); + + sb.append("updateTime=").append(updateTime) + .append(", longestDbLoadTimeInMs=").append(longestDbLoadTimeInMs) + .append(", Service-Version:").append(servicePolicies != null ? servicePolicies.getPolicyVersion() : "null") + .append(", Number-Of-Policies:").append(servicePolicies != null ? servicePolicies.getPolicies().size() : 0); + + sb.append("} "); + + return sb; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + } +} + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java b/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java new file mode 100644 index 0000000..e20cba8 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerServiceTagsCache.java @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.common; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.plugin.model.RangerServiceResource; +import org.apache.ranger.plugin.model.RangerTag; +import org.apache.ranger.plugin.store.TagStore; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.util.ServiceTags; + +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +public class RangerServiceTagsCache { + private static final Log LOG = LogFactory.getLog(RangerServiceTagsCache.class); + + private static volatile RangerServiceTagsCache sInstance = null; + private final boolean useServiceTagsCache; + private final int waitTimeInSeconds; + + private final Map<String, ServiceTagsWrapper> serviceTagsMap = new HashMap<String, ServiceTagsWrapper>(); + + public static RangerServiceTagsCache getInstance() { + if (sInstance == null) { + synchronized (RangerServiceTagsCache.class) { + if (sInstance == null) { + sInstance = new RangerServiceTagsCache(); + } + } + } + return sInstance; + } + + private RangerServiceTagsCache() { + useServiceTagsCache = RangerConfiguration.getInstance().getBoolean("ranger.admin.tag.download.usecache", true); + waitTimeInSeconds = RangerConfiguration.getInstance().getInt("ranger.admin.tag.download.cache.max.waittime.for.update", 20); + } + + public void dump() { + + if (useServiceTagsCache) { + Set<String> serviceNames = null; + + synchronized (this) { + serviceNames = serviceTagsMap.keySet(); + } + + if (CollectionUtils.isNotEmpty(serviceNames)) { + ServiceTagsWrapper cachedServiceTagsWrapper = null; + + for (String serviceName : serviceNames) { + cachedServiceTagsWrapper = serviceTagsMap.get(serviceName); + if (LOG.isDebugEnabled()) { + LOG.debug("serviceName:" + serviceName + ", Cached-MetaData:" + cachedServiceTagsWrapper); + } + } + } + } + } + + public ServiceTags getServiceTags(String serviceName) { + + ServiceTags ret = null; + + if (useServiceTagsCache && StringUtils.isNotBlank(serviceName)) { + ServiceTagsWrapper cachedServiceTagsWrapper = null; + synchronized (this) { + cachedServiceTagsWrapper = serviceTagsMap.get(serviceName); + } + if (cachedServiceTagsWrapper != null) { + ret = cachedServiceTagsWrapper.getServiceTags(); + } + } + + return ret; + } + + public ServiceTags getServiceTags(String serviceName, TagStore tagStore) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceTagsCache.getServiceTags(" + serviceName + ")"); + } + + ServiceTags ret = null; + + if (StringUtils.isNotBlank(serviceName)) { + + if (LOG.isDebugEnabled()) { + LOG.debug("useServiceTagsCache=" + useServiceTagsCache); + } + + ServiceTags serviceTags = null; + + if (!useServiceTagsCache) { + if (tagStore != null) { + try { + serviceTags = tagStore.getServiceTags(serviceName); + } catch (Exception exception) { + LOG.error("getServiceTags(" + serviceName + "): failed to get latest tags from tag-store", exception); + } + } else { + LOG.error("getServiceTags(" + serviceName + "): failed to get latest tags as tag-store is null!"); + } + } else { + ServiceTagsWrapper serviceTagsWrapper = null; + + synchronized (this) { + serviceTagsWrapper = serviceTagsMap.get(serviceName); + + if (serviceTagsWrapper == null) { + serviceTagsWrapper = new ServiceTagsWrapper(); + serviceTagsMap.put(serviceName, serviceTagsWrapper); + } + } + + if (tagStore != null) { + boolean refreshed = serviceTagsWrapper.getLatestOrCached(serviceName, tagStore); + LOG.info("tryRefreshFromStore returned " + refreshed); + } else { + LOG.error("getServiceTags(" + serviceName + "): failed to get latest tags as tag-store is null!"); + } + + serviceTags = serviceTagsWrapper.getServiceTags(); + } + + ret = serviceTags; + + } else { + LOG.error("getServiceTags() failed to get tags as serviceName is null or blank!"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceTagsCache.getServiceTags(" + serviceName + "): count=" + ((ret == null || ret.getTags() == null) ? 0 : ret.getTags().size())); + } + + return ret; + } + + private class ServiceTagsWrapper { + ServiceTags serviceTags; + Date updateTime = null; + long longestDbLoadTimeInMs = -1; + + ReentrantLock lock = new ReentrantLock(); + + ServiceTagsWrapper() { + serviceTags = null; + } + + ServiceTags getServiceTags() { + return serviceTags; + } + + Date getUpdateTime() { + return updateTime; + } + + long getLongestDbLoadTimeInMs() { + return longestDbLoadTimeInMs; + } + + boolean getLatestOrCached(String serviceName, TagStore tagStore) { + boolean ret = false; + + try { + ret = lock.tryLock(waitTimeInSeconds, TimeUnit.SECONDS); + if (ret) { + getLatest(serviceName, tagStore); + } + } catch (InterruptedException exception) { + LOG.error("tryRefreshFromStore:lock got interrupted..", exception); + } finally { + if (ret) { + lock.unlock(); + } + } + + return ret; + } + + void getLatest(String serviceName, TagStore tagStore) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> ServiceTagsWrapper.getLatest(" + serviceName + ")"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Found ServiceTags in-cache : " + (serviceTags != null)); + } + + Long tagVersionInDb = tagStore.getTagVersion(serviceName); + + + if (serviceTags == null || tagVersionInDb == null || !tagVersionInDb.equals(serviceTags.getTagVersion())) { + if (LOG.isDebugEnabled()) { + LOG.debug("loading serviceTags from db ... cachedServiceTagsVersion=" + (serviceTags != null ? serviceTags.getTagVersion() : null) + ", tagVersionInDb=" + tagVersionInDb); + } + + ServiceTags serviceTagsFromDb = null; + + try { + long startTimeMs = System.currentTimeMillis(); + + serviceTagsFromDb = tagStore.getServiceTags(serviceName); + + long dbLoadTime = System.currentTimeMillis() - startTimeMs; + + if (dbLoadTime > longestDbLoadTimeInMs) { + longestDbLoadTimeInMs = dbLoadTime; + } + updateTime = new Date(); + } catch (Exception exception) { + LOG.error("getServiceTags(" + serviceName + "): failed to get latest tags from tag-store", exception); + } + + if (serviceTagsFromDb != null) { + if (serviceTagsFromDb.getTagVersion() == null) { + serviceTagsFromDb.setTagVersion(0L); + } + serviceTags = serviceTagsFromDb; + pruneUnusedAttributes(); + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== ServiceTagsWrapper.getLatest(" + serviceName + ")"); + } + } + + private void pruneUnusedAttributes() { + if (serviceTags != null) { + serviceTags.setOp(null); + serviceTags.setTagModel(null); + serviceTags.setTagUpdateTime(null); + + serviceTags.setTagDefinitions(null); + + for (Map.Entry<Long, RangerTag> entry : serviceTags.getTags().entrySet()) { + RangerTag tag = entry.getValue(); + tag.setCreatedBy(null); + tag.setCreateTime(null); + tag.setUpdatedBy(null); + tag.setUpdateTime(null); + tag.setGuid(null); + } + + for (RangerServiceResource serviceResource : serviceTags.getServiceResources()) { + serviceResource.setCreatedBy(null); + serviceResource.setCreateTime(null); + serviceResource.setUpdatedBy(null); + serviceResource.setUpdateTime(null); + serviceResource.setGuid(null); + + serviceResource.setServiceName(null); + serviceResource.setResourceSignature(null); + } + } + } + + StringBuilder toString(StringBuilder sb) { + sb.append("RangerServiceTagsWrapper={"); + + sb.append("updateTime=").append(updateTime) + .append(", longestDbLoadTimeInMs=").append(longestDbLoadTimeInMs) + .append(", Service-Version:").append(serviceTags != null ? serviceTags.getTagVersion() : "null") + .append(", Number-Of-Tags:").append(serviceTags != null ? serviceTags.getTags().size() : 0); + + sb.append("} "); + + return sb; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + } +} + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4e2e83ee/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 2464063..174a5ee 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -1927,10 +1927,11 @@ public class ServiceREST { String propertyPrefix = "ranger.admin"; - options.evaluatorType = RangerConfiguration.getInstance().get(propertyPrefix + ".policyengine.option.evaluator.type", RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED); + options.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED; options.cacheAuditResults = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", false); options.disableContextEnrichers = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", true); options.disableCustomConditions = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.custom.conditions", true); + options.evaluateDelegateAdminOnly = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.evaluate.delegateadmin.only", true); RangerPolicyEngineCache.getInstance().setPolicyEngineOptions(options);; }
