RANGER-2165: Address JPA Cache issue when policies Create, Update and Delete are done via REST API in Apache Ranger admin
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/1a358571 Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/1a358571 Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/1a358571 Branch: refs/heads/ranger-1.1 Commit: 1a3585717f2c27b100a6dd3dda4198da4ee1ec33 Parents: 4a13ff7 Author: Abhay Kulkarni <[email protected]> Authored: Tue Jul 31 15:15:33 2018 -0700 Committer: Mehul Parikh <[email protected]> Committed: Tue Sep 4 11:29:46 2018 +0530 ---------------------------------------------------------------------- .../ranger/biz/RangerPolicyRetriever.java | 82 ++++++++++++++-- .../apache/ranger/biz/RangerTagDBRetriever.java | 99 ++++++++++++++++++-- .../org/apache/ranger/biz/ServiceDBStore.java | 2 +- .../java/org/apache/ranger/biz/TagDBStore.java | 8 +- 4 files changed, 175 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ranger/blob/1a358571/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java index 463957c..3e291d5 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerPolicyRetriever.java @@ -47,17 +47,36 @@ import org.apache.ranger.plugin.model.RangerValiditySchedule; import org.apache.ranger.plugin.policyevaluator.RangerPolicyItemEvaluator; import org.apache.ranger.plugin.util.RangerPerfTracer; import org.apache.ranger.service.RangerPolicyService; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionTemplate; public class RangerPolicyRetriever { static final Log LOG = LogFactory.getLog(RangerPolicyRetriever.class); static final Log PERF_LOG = RangerPerfTracer.getPerfLogger("db.RangerPolicyRetriever"); - final RangerDaoManager daoMgr; - final LookupCache lookupCache; + private final RangerDaoManager daoMgr; + private final LookupCache lookupCache = new LookupCache(); + + private final PlatformTransactionManager txManager; + private final TransactionTemplate txTemplate; + + public RangerPolicyRetriever(RangerDaoManager daoMgr, PlatformTransactionManager txManager) { + this.daoMgr = daoMgr; + this.txManager = txManager; + if (this.txManager != null) { + this.txTemplate = new TransactionTemplate(this.txManager); + this.txTemplate.setReadOnly(true); + } else { + this.txTemplate = null; + } + } public RangerPolicyRetriever(RangerDaoManager daoMgr) { this.daoMgr = daoMgr; - this.lookupCache = new LookupCache(); + this.txManager = null; + this.txTemplate = null; } public List<RangerPolicy> getServicePolicies(Long serviceId) { @@ -96,7 +115,41 @@ public class RangerPolicyRetriever { return ret; } - public List<RangerPolicy> getServicePolicies(XXService xService) { + private class PolicyLoaderThread extends Thread { + final TransactionTemplate txTemplate; + final XXService xService; + List<RangerPolicy> policies; + + PolicyLoaderThread(TransactionTemplate txTemplate, final XXService xService) { + this.txTemplate = txTemplate; + this.xService = xService; + } + + public List<RangerPolicy> getPolicies() { return policies; } + + @Override + public void run() { + try { + policies = txTemplate.execute(new TransactionCallback<List<RangerPolicy>>() { + @Override + public List<RangerPolicy> doInTransaction(TransactionStatus status) { + try { + RetrieverContext ctx = new RetrieverContext(xService); + return ctx.getAllPolicies(); + } catch (Exception ex) { + LOG.error("RangerPolicyRetriever.getServicePolicies(): Failed to get policies for service:[" + xService.getName() + "] in a new transaction", ex); + status.setRollbackOnly(); + return null; + } + } + }); + } catch (Throwable ex) { + LOG.error("RangerPolicyRetriever.getServicePolicies(): Failed to get policies for service:[" + xService.getName() + "] in a new transaction", ex); + } + } + } + + public List<RangerPolicy> getServicePolicies(final XXService xService) { String serviceName = xService == null ? null : xService.getName(); Long serviceId = xService == null ? null : xService.getId(); @@ -112,9 +165,26 @@ public class RangerPolicyRetriever { } if(xService != null) { - RetrieverContext ctx = new RetrieverContext(xService); + if (txTemplate == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Transaction Manager is null; Retrieving policies in the existing transaction"); + } + RetrieverContext ctx = new RetrieverContext(xService); + ret = ctx.getAllPolicies(); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Retrieving policies in a new, read-only transaction"); + } - ret = ctx.getAllPolicies(); + PolicyLoaderThread t = new PolicyLoaderThread(txTemplate, xService); + t.start(); + try { + t.join(); + ret = t.getPolicies(); + } catch (InterruptedException ie) { + LOG.error("Failed to retrieve policies in a new, read-only thread.", ie); + } + } } else { if(LOG.isDebugEnabled()) { LOG.debug("RangerPolicyRetriever.getServicePolicies(xService=" + xService + "): invalid parameter"); http://git-wip-us.apache.org/repos/asf/ranger/blob/1a358571/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java index 5550572..789068f 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerTagDBRetriever.java @@ -38,6 +38,10 @@ import org.apache.ranger.entity.*; import org.apache.ranger.plugin.model.*; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; import org.apache.ranger.plugin.util.RangerPerfTracer; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionTemplate; public class RangerTagDBRetriever { static final Log LOG = LogFactory.getLog(RangerTagDBRetriever.class); @@ -48,6 +52,9 @@ public class RangerTagDBRetriever { private final XXService xService; private final LookupCache lookupCache; + private final PlatformTransactionManager txManager; + private final TransactionTemplate txTemplate; + private List<RangerServiceResource> serviceResources; private Map<Long, RangerTagDef> tagDefs; private Map<Long, RangerTag> tags; @@ -55,8 +62,15 @@ public class RangerTagDBRetriever { private boolean filterForServicePlugin; - public RangerTagDBRetriever(final RangerDaoManager daoMgr, final XXService xService) { + public RangerTagDBRetriever(final RangerDaoManager daoMgr, final PlatformTransactionManager txManager, final XXService xService) { this.daoMgr = daoMgr; + this.txManager = txManager; + if (this.txManager != null) { + this.txTemplate = new TransactionTemplate(this.txManager); + this.txTemplate.setReadOnly(true); + } else { + this.txTemplate = null; + } this.xService = xService; this.lookupCache = new LookupCache(); @@ -70,14 +84,27 @@ public class RangerTagDBRetriever { } filterForServicePlugin = RangerConfiguration.getInstance().getBoolean(OPTION_RANGER_FILTER_TAGS_FOR_SERVICE_PLUGIN, false); - TagRetrieverServiceResourceContext serviceResourceContext = new TagRetrieverServiceResourceContext(xService); - TagRetrieverTagDefContext tagDefContext = new TagRetrieverTagDefContext(xService); - TagRetrieverTagContext tagContext = new TagRetrieverTagContext(xService); - serviceResources = serviceResourceContext.getAllServiceResources(); - tagDefs = tagDefContext.getAllTagDefs(); - tags = tagContext.getAllTags(); - tagResourceMaps = getAllTagResourceMaps(); + if (this.txTemplate == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Load Tags in the same thread and using an existing transaction"); + } + if (initializeTagCache(xService) == false) { + LOG.error("Failed to get tags for service:[" + xService.getName() + "] in the same thread and using an existing transaction"); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Load Tags in a separate thread and using a new transaction"); + } + + TagLoaderThread t = new TagLoaderThread(txTemplate, xService); + t.start(); + try { + t.join(); + } catch (InterruptedException ie) { + LOG.error("Failed to get Tags in a separate thread and using a new transaction", ie); + } + } RangerPerfTracer.log(perf); @@ -100,6 +127,30 @@ public class RangerTagDBRetriever { return tags; } + private boolean initializeTagCache(XXService xService) { + boolean ret; + try { + TagRetrieverServiceResourceContext serviceResourceContext = new TagRetrieverServiceResourceContext(xService); + TagRetrieverTagDefContext tagDefContext = new TagRetrieverTagDefContext(xService); + TagRetrieverTagContext tagContext = new TagRetrieverTagContext(xService); + + serviceResources = serviceResourceContext.getAllServiceResources(); + tagDefs = tagDefContext.getAllTagDefs(); + tags = tagContext.getAllTags(); + + tagResourceMaps = getAllTagResourceMaps(); + + ret = true; + } catch (Exception ex) { + LOG.error("Failed to get tags for service:[" + xService.getName() + "]"); + serviceResources = null; + tagDefs = null; + tags = null; + tagResourceMaps = null; + ret = false; + } + return ret; + } private List<RangerTagResourceMap> getAllTagResourceMaps() { List<XXTagResourceMap> xTagResourceMaps = filterForServicePlugin ? daoMgr.getXXTagResourceMap().findForServicePlugin(xService.getId()) : daoMgr.getXXTagResourceMap().findByServiceId(xService.getId()); @@ -200,6 +251,38 @@ public class RangerTagDBRetriever { } } + private class TagLoaderThread extends Thread { + final TransactionTemplate txTemplate; + final XXService xService; + + TagLoaderThread(TransactionTemplate txTemplate, final XXService xService) { + this.txTemplate = txTemplate; + this.xService = xService; + } + + @Override + public void run() { + try { + Boolean result = txTemplate.execute(new TransactionCallback<Boolean>() { + @Override + public Boolean doInTransaction(TransactionStatus status) { + boolean ret = initializeTagCache(xService); + if (!ret) { + status.setRollbackOnly(); + LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction"); + } + return ret; + } + }); + if (LOG.isDebugEnabled()) { + LOG.debug("transaction result:[" + result +"]"); + } + } catch (Throwable ex) { + LOG.error("Failed to get tags for service:[" + xService.getName() + "] in a new transaction", ex); + } + } + } + private class TagRetrieverServiceResourceContext { final XXService service; http://git-wip-us.apache.org/repos/asf/ranger/blob/1a358571/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index e75ea68..0773616 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -2462,7 +2462,7 @@ public class ServiceDBStore extends AbstractServiceStore { LOG.debug("==> ServiceDBStore.getServicePoliciesFromDb(" + service.getName() + ")"); } - RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr); + RangerPolicyRetriever policyRetriever = new RangerPolicyRetriever(daoMgr, txManager); List<RangerPolicy> ret = policyRetriever.getServicePolicies(service); http://git-wip-us.apache.org/repos/asf/ranger/blob/1a358571/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java index 8341a73..d29df93 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java @@ -64,7 +64,9 @@ import org.apache.ranger.service.RangerTagResourceMapService; import org.apache.ranger.service.RangerTagService; import org.apache.ranger.service.RangerServiceResourceService; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; import javax.annotation.PostConstruct; import javax.servlet.http.HttpServletResponse; @@ -89,6 +91,10 @@ public class TagDBStore extends AbstractTagStore { RangerDaoManager daoManager; @Autowired + @Qualifier(value = "transactionManager") + PlatformTransactionManager txManager; + + @Autowired RESTErrorUtil errorUtil; @Autowired @@ -991,7 +997,7 @@ public class TagDBStore extends AbstractTagStore { throw new Exception("service-def does not exist. id=" + xxService.getType()); } - RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, xxService); + RangerTagDBRetriever tagDBRetriever = new RangerTagDBRetriever(daoManager, txManager, xxService); Map<Long, RangerTagDef> tagDefMap = tagDBRetriever.getTagDefs(); Map<Long, RangerTag> tagMap = tagDBRetriever.getTags();
