Repository: incubator-ranger Updated Branches: refs/heads/master 23f540e56 -> c38a718ed
RANGER-997 : Improve Policy Listing performance on Reports page having more than 1000 policies Signed-off-by: Velmurugan Periasamy <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/c38a718e Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/c38a718e Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/c38a718e Branch: refs/heads/master Commit: c38a718ede6d9888e6c1a413a66f26df0b45a04e Parents: 23f540e Author: pradeep agrawal <[email protected]> Authored: Tue Jun 7 20:00:55 2016 +0530 Committer: Velmurugan Periasamy <[email protected]> Committed: Thu Jun 9 09:02:28 2016 -0400 ---------------------------------------------------------------------- .../org/apache/ranger/biz/ServiceDBStore.java | 96 +++++++++++++++++++- .../ranger/service/RangerBaseModelService.java | 2 +- .../ranger/service/RangerPolicyServiceBase.java | 52 ----------- .../apache/ranger/biz/TestServiceDBStore.java | 13 --- .../service/TestRangerPolicyServiceBase.java | 36 -------- 5 files changed, 93 insertions(+), 106 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c38a718e/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index a659d45..bf03e30 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -22,10 +22,14 @@ package org.apache.ranger.biz; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.Map.Entry; import java.util.StringTokenizer; import java.io.ByteArrayInputStream; @@ -1905,7 +1909,7 @@ public class ServiceDBStore extends AbstractServiceStore { if(LOG.isDebugEnabled()) { LOG.debug("==> ServiceDBStore.getPolicies()"); } - RangerPolicyList policyList = policyService.searchRangerPolicies(filter); + RangerPolicyList policyList = searchRangerPolicies(filter); List<RangerPolicy> ret = policyList.getPolicies(); if(LOG.isDebugEnabled()) { LOG.debug("<== ServiceDBStore.getPolicies()"); @@ -1926,7 +1930,7 @@ public class ServiceDBStore extends AbstractServiceStore { if (!CollectionUtils.isEmpty(serviceTypeList)) { for (String serviceType : serviceTypeList) { filter.setParam("serviceType", serviceType); - RangerPolicyList policyList = policyService.searchRangerPolicies(filter); + RangerPolicyList policyList = searchRangerPolicies(filter); if (policyList!=null){ retTemp = policyList.getPolicies(); if(!CollectionUtils.isEmpty(retTemp)) { @@ -1947,7 +1951,7 @@ public class ServiceDBStore extends AbstractServiceStore { } } } else { - RangerPolicyList policyList = policyService.searchRangerPolicies(filter); + RangerPolicyList policyList = searchRangerPolicies(filter); ret = policyList.getPolicies(); if (!CollectionUtils.isEmpty(ret)) { for (RangerPolicy policy : ret) { @@ -2015,7 +2019,7 @@ public class ServiceDBStore extends AbstractServiceStore { LOG.debug("==> ServiceDBStore.getPaginatedPolicies(+ " + filter + ")"); } - RangerPolicyList policyList = policyService.searchRangerPolicies(filter); + RangerPolicyList policyList = searchRangerPolicies(filter); if (LOG.isDebugEnabled()) { LOG.debug("before filter: count=" + policyList.getListSize()); @@ -3627,4 +3631,88 @@ public class ServiceDBStore extends AbstractServiceStore { cellStatus.setCellStyle(cellStyle); cellStatus.setCellValue("Status"); } + + + private RangerPolicyList searchRangerPolicies(SearchFilter searchFilter) { + List<RangerPolicy> policyList = new ArrayList<RangerPolicy>(); + RangerPolicyList retList = new RangerPolicyList(); + Map<Long,RangerPolicy> policyMap=new HashMap<Long,RangerPolicy>(); + Set<Long> processedServices=new HashSet<Long>(); + Set<Long> processedPolicies=new HashSet<Long>(); + Comparator<RangerPolicy> comparator = new Comparator<RangerPolicy>() { + public int compare(RangerPolicy c1, RangerPolicy c2) { + return (int) ((c1.getId()).compareTo(c2.getId())); + } + }; + + List<XXPolicy> xPolList = (List<XXPolicy>) policyService.searchResources(searchFilter, policyService.searchFields, policyService.sortFields, retList); + if (!CollectionUtils.isEmpty(xPolList)) { + for (XXPolicy xXPolicy : xPolList) { + if(!processedServices.contains(xXPolicy.getService())){ + loadRangerPolicies(xXPolicy.getService(),processedServices,policyMap); + } + } + } + String userName = searchFilter.getParam("user"); + if (!StringUtils.isEmpty(userName)) { + searchFilter.removeParam("user"); + Set<String> groupNames = daoMgr.getXXGroupUser().findGroupNamesByUserName(userName); + if (!CollectionUtils.isEmpty(groupNames)) { + List<XXPolicy> xPolList2 = null; + for (String groupName : groupNames) { + xPolList2 = new ArrayList<XXPolicy>(); + searchFilter.setParam("group", groupName); + xPolList2 = (List<XXPolicy>) policyService.searchResources(searchFilter, policyService.searchFields, policyService.sortFields, retList); + if (!CollectionUtils.isEmpty(xPolList2)) { + for (XXPolicy xPol2 : xPolList2) { + if(xPol2!=null){ + if(!processedPolicies.contains(xPol2.getId())){ + if(!processedServices.contains(xPol2.getService())){ + loadRangerPolicies(xPol2.getService(),processedServices,policyMap); + } + if(policyMap.containsKey(xPol2.getId())){ + policyList.add(policyMap.get(xPol2.getId())); + processedPolicies.add(xPol2.getId()); + } + } + } + } + } + } + } + } + if (!CollectionUtils.isEmpty(xPolList)) { + for (XXPolicy xPol : xPolList) { + if(xPol!=null){ + if(!processedPolicies.contains(xPol.getId())){ + if(!processedServices.contains(xPol.getService())){ + loadRangerPolicies(xPol.getService(),processedServices,policyMap); + } + if(policyMap.containsKey(xPol.getId())){ + policyList.add(policyMap.get(xPol.getId())); + processedPolicies.add(xPol.getId()); + } + } + } + } + Collections.sort(policyList, comparator); + } + retList.setPolicies(policyList); + return retList; + } + + private void loadRangerPolicies(Long serviceId,Set<Long> processedServices,Map<Long,RangerPolicy> policyMap){ + try { + List<RangerPolicy> tempPolicyList = getServicePolicies(serviceId,null); + if(!CollectionUtils.isEmpty(tempPolicyList)){ + for (RangerPolicy rangerPolicy : tempPolicyList) { + if(!policyMap.containsKey(rangerPolicy.getId())){ + policyMap.put(rangerPolicy.getId(), rangerPolicy); + } + } + processedServices.add(serviceId); + } + } catch (Exception e) { + } + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c38a718e/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java index 2d5440d..07765e0 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerBaseModelService.java @@ -307,7 +307,7 @@ public abstract class RangerBaseModelService<T extends XXDBBase, V extends Range * */ - protected List<T> searchResources(SearchFilter searchCriteria, + public List<T> searchResources(SearchFilter searchCriteria, List<SearchField> searchFieldList, List<SortField> sortFieldList, VList vList) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c38a718e/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java index bde18bd..b7598ea 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java @@ -17,14 +17,7 @@ package org.apache.ranger.service; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Set; - import org.apache.commons.lang.StringUtils; -import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.common.GUIDUtil; import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.SearchField; @@ -32,14 +25,11 @@ import org.apache.ranger.common.SortField; import org.apache.ranger.common.SearchField.DATA_TYPE; import org.apache.ranger.common.SearchField.SEARCH_TYPE; import org.apache.ranger.common.SortField.SORT_ORDER; -import org.apache.ranger.entity.XXPolicy; import org.apache.ranger.entity.XXPolicyBase; import org.apache.ranger.entity.XXService; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.util.SearchFilter; -import org.apache.ranger.view.RangerPolicyList; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.util.CollectionUtils; public abstract class RangerPolicyServiceBase<T extends XXPolicyBase, V extends RangerPolicy> extends RangerBaseModelService<T, V> { @@ -120,46 +110,4 @@ public abstract class RangerPolicyServiceBase<T extends XXPolicyBase, V extends vObj.setIsAuditEnabled(xObj.getIsAuditEnabled()); return vObj; } - - @SuppressWarnings("unchecked") - public RangerPolicyList searchRangerPolicies(SearchFilter searchFilter) { - List<RangerPolicy> policyList = new ArrayList<RangerPolicy>(); - RangerPolicyList retList = new RangerPolicyList(); - - Comparator<RangerPolicy> comparator = new Comparator<RangerPolicy>() { - public int compare(RangerPolicy c1, RangerPolicy c2) { - return (int) ((c1.getId()).compareTo(c2.getId())); - } - }; - - List<XXPolicy> xPolList = (List<XXPolicy>) searchResources(searchFilter, searchFields, sortFields, retList); - String userName = searchFilter.getParam("user"); - if (!StringUtil.isEmpty(userName)) { - searchFilter.removeParam("user"); - Set<String> groupNames = daoMgr.getXXGroupUser().findGroupNamesByUserName(userName); - if (!CollectionUtils.isEmpty(groupNames)) { - List<XXPolicy> xPolList2 = null; - for (String groupName : groupNames) { - xPolList2 = new ArrayList<XXPolicy>(); - searchFilter.setParam("group", groupName); - xPolList2 = (List<XXPolicy>) searchResources(searchFilter, searchFields, sortFields, retList); - if (!CollectionUtils.isEmpty(xPolList2)) { - for (XXPolicy xPol2 : xPolList2) { - if(xPol2!=null){ - policyList.add(populateViewBean((T) xPol2)); - } - } - } - } - } - } - if (!CollectionUtils.isEmpty(xPolList)) { - for (XXPolicy xPol : xPolList) { - policyList.add(populateViewBean((T) xPol)); - } - Collections.sort(policyList, comparator); - } - retList.setPolicies(policyList); - return retList; - } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c38a718e/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java ---------------------------------------------------------------------- diff --git a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java index a3cd613..ea1563e 100644 --- a/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java +++ b/security-admin/src/test/java/org/apache/ranger/biz/TestServiceDBStore.java @@ -2462,11 +2462,8 @@ public class TestServiceDBStore { policyListObj.setStartIndex(0); policyListObj.setTotalCount(10); - Mockito.when(policyService.searchRangerPolicies(filter)).thenReturn( - policyListObj); List<RangerPolicy> dbRangerPolicy = serviceDBStore.getPolicies(filter); Assert.assertNotNull(dbRangerPolicy); - Mockito.verify(policyService).searchRangerPolicies(filter); } @Test @@ -2483,13 +2480,9 @@ public class TestServiceDBStore { policyListObj.setStartIndex(0); policyListObj.setTotalCount(10); - Mockito.when(policyService.searchRangerPolicies(filter)).thenReturn( - policyListObj); - PList<RangerPolicy> dbRangerPolicyList = serviceDBStore .getPaginatedPolicies(filter); Assert.assertNotNull(dbRangerPolicyList); - Mockito.verify(policyService).searchRangerPolicies(filter); } @Test @@ -2596,14 +2589,9 @@ public class TestServiceDBStore { filter.setParam(SearchFilter.POLICY_NAME, "policyName"); filter.setParam(SearchFilter.SERVICE_NAME, "serviceName"); - Mockito.when(policyService.searchRangerPolicies(filter)).thenReturn( - policyList); - - PList<RangerPolicy> dbRangerPolicyList = serviceDBStore .getPaginatedServicePolicies(serviceName, filter); Assert.assertNotNull(dbRangerPolicyList); - Mockito.verify(policyService).searchRangerPolicies(filter); } @Test @@ -2619,7 +2607,6 @@ public class TestServiceDBStore { Mockito.when(daoManager.getXXService()).thenReturn(xServiceDao); Mockito.when(xServiceDao.getById(Id)).thenReturn(xService); - thrown.expect(Exception.class); PList<RangerPolicy> dbRangerPolicyList = serviceDBStore .getPaginatedServicePolicies(rangerService.getId(), filter); Mockito.verify(daoManager).getXXService(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/c38a718e/security-admin/src/test/java/org/apache/ranger/service/TestRangerPolicyServiceBase.java ---------------------------------------------------------------------- diff --git a/security-admin/src/test/java/org/apache/ranger/service/TestRangerPolicyServiceBase.java b/security-admin/src/test/java/org/apache/ranger/service/TestRangerPolicyServiceBase.java index 75e2ebd..7910cbd 100644 --- a/security-admin/src/test/java/org/apache/ranger/service/TestRangerPolicyServiceBase.java +++ b/security-admin/src/test/java/org/apache/ranger/service/TestRangerPolicyServiceBase.java @@ -21,18 +21,13 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; - -import javax.persistence.EntityManager; import javax.ws.rs.WebApplicationException; - import org.apache.ranger.biz.RangerBizUtil; import org.apache.ranger.common.ContextUtil; import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.RESTErrorUtil; import org.apache.ranger.common.RangerSearchUtil; -import org.apache.ranger.common.SearchField; import org.apache.ranger.common.UserSessionBase; -import org.apache.ranger.common.db.BaseDao; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.db.XXServiceDao; import org.apache.ranger.entity.XXDBBase; @@ -43,10 +38,8 @@ import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; -import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.security.context.RangerContextHolder; import org.apache.ranger.security.context.RangerSecurityContext; -import org.apache.ranger.view.RangerPolicyList; import org.junit.Assert; import org.junit.FixMethodOrder; import org.junit.Rule; @@ -243,33 +236,4 @@ public class TestRangerPolicyServiceBase { Mockito.verify(daoManager).getXXService(); } - @Test - public void test4searchRangerPolicies() { - EntityManager entityManager = Mockito.mock(EntityManager.class); - SearchFilter searchFilter = new SearchFilter(); - searchFilter.setParam(SearchFilter.POLICY_NAME, "policyName"); - searchFilter.setParam(SearchFilter.SERVICE_NAME, "serviceName"); - - String searchString = "policyName"; - String sortString = "asc"; - List<SearchField> searchFieldList = new ArrayList<SearchField>(); - boolean isCountQuery = false; - - BaseDao baseDao = Mockito.mock(BaseDao.class); - - Mockito.when(daoManager.getDaoForClassName(Mockito.anyString())) - .thenReturn(baseDao); - Mockito.when(daoManager.getEntityManager()).thenReturn(entityManager); - Mockito.when( - searchUtil - .createSearchQuery(entityManager, searchString, - sortString, searchFilter, searchFieldList, - isCountQuery)).thenReturn(null); - - RangerPolicyList dbRangerPolicyList = policyService - .searchRangerPolicies(searchFilter); - Assert.assertNotNull(dbRangerPolicyList); - Mockito.verify(daoManager).getDaoForClassName(Mockito.anyString()); - } - }
