RANGER-1025 : Policy search REST API implemented in public api V1 is not returning results as expected
Signed-off-by: Velmurugan Periasamy <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/07696a70 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/07696a70 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/07696a70 Branch: refs/heads/master Commit: 07696a705cea966c4a0b8bfdc85f190e45bc2515 Parents: 7e5e970 Author: pradeep agrawal <[email protected]> Authored: Mon Jun 13 09:56:13 2016 +0530 Committer: Velmurugan Periasamy <[email protected]> Committed: Fri Jun 17 00:11:00 2016 -0400 ---------------------------------------------------------------------- .../org/apache/ranger/biz/ServiceDBStore.java | 10 ++-- .../apache/ranger/common/RangerSearchUtil.java | 1 + .../org/apache/ranger/common/ServiceUtil.java | 51 ++++++++++++-------- .../java/org/apache/ranger/rest/PublicAPIs.java | 17 +++++-- .../ranger/service/RangerPolicyServiceBase.java | 2 + 5 files changed, 51 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/07696a70/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 739ad05..794de71 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -3651,7 +3651,7 @@ public class ServiceDBStore extends AbstractServiceStore { if (!CollectionUtils.isEmpty(xPolList)) { for (XXPolicy xXPolicy : xPolList) { if(!processedServices.contains(xXPolicy.getService())){ - loadRangerPolicies(xXPolicy.getService(),processedServices,policyMap); + loadRangerPolicies(xXPolicy.getService(),processedServices,policyMap,searchFilter); } } } @@ -3670,7 +3670,7 @@ public class ServiceDBStore extends AbstractServiceStore { if(xPol2!=null){ if(!processedPolicies.contains(xPol2.getId())){ if(!processedServices.contains(xPol2.getService())){ - loadRangerPolicies(xPol2.getService(),processedServices,policyMap); + loadRangerPolicies(xPol2.getService(),processedServices,policyMap,searchFilter); } if(policyMap.containsKey(xPol2.getId())){ policyList.add(policyMap.get(xPol2.getId())); @@ -3688,7 +3688,7 @@ public class ServiceDBStore extends AbstractServiceStore { if(xPol!=null){ if(!processedPolicies.contains(xPol.getId())){ if(!processedServices.contains(xPol.getService())){ - loadRangerPolicies(xPol.getService(),processedServices,policyMap); + loadRangerPolicies(xPol.getService(),processedServices,policyMap,searchFilter); } if(policyMap.containsKey(xPol.getId())){ policyList.add(policyMap.get(xPol.getId())); @@ -3703,9 +3703,9 @@ public class ServiceDBStore extends AbstractServiceStore { return retList; } - private void loadRangerPolicies(Long serviceId,Set<Long> processedServices,Map<Long,RangerPolicy> policyMap){ + private void loadRangerPolicies(Long serviceId,Set<Long> processedServices,Map<Long,RangerPolicy> policyMap,SearchFilter searchFilter){ try { - List<RangerPolicy> tempPolicyList = getServicePolicies(serviceId,null); + List<RangerPolicy> tempPolicyList = getServicePolicies(serviceId,searchFilter); if(!CollectionUtils.isEmpty(tempPolicyList)){ for (RangerPolicy rangerPolicy : tempPolicyList) { if(!policyMap.containsKey(rangerPolicy.getId())){ http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/07696a70/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java index b8f7c88..2ad5795 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java @@ -136,6 +136,7 @@ public class RangerSearchUtil extends SearchUtil { ret.setParam(SearchFilter.GROUP, request.getParameter("groupName")); ret.setParam(SearchFilter.IS_ENABLED, request.getParameter("isEnabled")); ret.setParam(SearchFilter.IS_RECURSIVE, request.getParameter("isRecursive")); + ret.setParam(SearchFilter.POL_RESOURCE, request.getParameter(SearchFilter.POL_RESOURCE)); ret.setParam(SearchFilter.RESOURCE_PREFIX + "path", request.getParameter("resourceName")); ret.setParam(SearchFilter.RESOURCE_PREFIX + "database", request.getParameter("databases")); ret.setParam(SearchFilter.RESOURCE_PREFIX + "table", request.getParameter("tables")); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/07696a70/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java b/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java index 0feb5db..5d0551a 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java @@ -50,6 +50,7 @@ import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; import org.apache.ranger.plugin.util.GrantRevokeRequest; +import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.view.VXAsset; import org.apache.ranger.view.VXAuditMap; import org.apache.ranger.view.VXDataObject; @@ -1153,33 +1154,43 @@ public class ServiceUtil { } - public VXPolicyList rangerPolicyListToPublic(List<RangerPolicy> rangerPolicyList) { + public VXPolicyList rangerPolicyListToPublic(List<RangerPolicy> rangerPolicyList,SearchFilter filter) { RangerService service = null; List<VXPolicy> vXPolicyList = new ArrayList<VXPolicy>(); - for ( RangerPolicy policy : rangerPolicyList) { - try { - service = svcStore.getServiceByName(policy.getService()); - } catch(Exception excp) { - throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); - } - - if(service == null) { - throw restErrorUtil.createRESTException(HttpServletResponse.SC_NOT_FOUND, "Not found", true); - } - - VXPolicy vXPolicy = toVXPolicy(policy,service); - - if(vXPolicy != null) { - vXPolicyList.add(vXPolicy); + VXPolicyList vXPolicyListObj = new VXPolicyList(new ArrayList<VXPolicy>()); + if(CollectionUtils.isNotEmpty(rangerPolicyList)) { + int totalCount = rangerPolicyList.size(); + int startIndex = filter == null ? 0 : filter.getStartIndex(); + int pageSize = filter == null ? totalCount : filter.getMaxRows(); + int toIndex = Math.min(startIndex + pageSize, totalCount); + String sortType = filter == null ? null : filter.getSortType(); + String sortBy = filter == null ? null : filter.getSortBy(); + for(int i = startIndex; i < toIndex; i++) { + RangerPolicy policy =rangerPolicyList.get(i); + try { + service = svcStore.getServiceByName(policy.getService()); + } catch(Exception excp) { + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); + } + if(service == null) { + throw restErrorUtil.createRESTException(HttpServletResponse.SC_NOT_FOUND, "Not found", true); + } + VXPolicy vXPolicy = toVXPolicy(policy,service); + if(vXPolicy != null) { + vXPolicyList.add(vXPolicy); + } } + vXPolicyListObj = new VXPolicyList(vXPolicyList); + vXPolicyListObj.setPageSize(pageSize); + vXPolicyListObj.setResultSize(vXPolicyList.size()); + vXPolicyListObj.setStartIndex(startIndex); + vXPolicyListObj.setTotalCount(totalCount); + vXPolicyListObj.setSortBy(sortBy); + vXPolicyListObj.setSortType(sortType); } - - VXPolicyList vXPolicyListObj = new VXPolicyList(vXPolicyList); - return vXPolicyListObj; - } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/07696a70/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIs.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIs.java b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIs.java index 21fdcd1..1f465d5 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIs.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIs.java @@ -348,19 +348,26 @@ public class PublicAPIs { } SearchFilter filter = searchUtil.getSearchFilterFromLegacyRequest(request, policyService.sortFields); + // get all policies from the store; pick the page to return after applying filter + int savedStartIndex = filter == null ? 0 : filter.getStartIndex(); + int savedMaxRows = filter == null ? Integer.MAX_VALUE : filter.getMaxRows(); + if(filter != null) { + filter.setStartIndex(0); + filter.setMaxRows(Integer.MAX_VALUE); + } List<RangerPolicy> rangerPolicyList = serviceREST.getPolicies(filter); - + if(filter != null) { + filter.setStartIndex(savedStartIndex); + filter.setMaxRows(savedMaxRows); + } VXPolicyList vXPolicyList = null; - if (rangerPolicyList != null) { - vXPolicyList = serviceUtil.rangerPolicyListToPublic(rangerPolicyList); + vXPolicyList = serviceUtil.rangerPolicyListToPublic(rangerPolicyList,filter); } - if(logger.isDebugEnabled()) { logger.debug("<== PublicAPIs.searchPolicies(): " + vXPolicyList ); } - return vXPolicyList; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/07696a70/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java index b7598ea..2649ff3 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerPolicyServiceBase.java @@ -49,6 +49,8 @@ public abstract class RangerPolicyServiceBase<T extends XXPolicyBase, V extends "XXService xSvc", "xSvc.id = obj.service")); searchFields .add(new SearchField(SearchFilter.IS_ENABLED, "obj.isEnabled", DATA_TYPE.BOOLEAN, SEARCH_TYPE.FULL)); + searchFields.add(new SearchField(SearchFilter.IS_RECURSIVE,"xPolRes.isRecursive",DATA_TYPE.BOOLEAN,SEARCH_TYPE.FULL, + "XXPolicyResource xPolRes","obj.id=xPolRes.policyId")); searchFields.add(new SearchField(SearchFilter.POLICY_ID, "obj.id", DATA_TYPE.INTEGER, SEARCH_TYPE.FULL)); searchFields.add(new SearchField(SearchFilter.POLICY_NAME, "obj.name", DATA_TYPE.STRING, SEARCH_TYPE.FULL)); searchFields.add(new SearchField(SearchFilter.GUID, "obj.guid", DATA_TYPE.STRING, SEARCH_TYPE.FULL));
