This is an automated email from the ASF dual-hosted git repository.
pradeep pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new c5bf2f6 RANGER-2528: Export API to get zone, unzone as well as tag
based policies from Ranger.
c5bf2f6 is described below
commit c5bf2f6364a97539451656d28fd36e35d8e2736d
Author: Sanjar Matin <[email protected]>
AuthorDate: Sat Sep 21 18:10:14 2019 +0530
RANGER-2528: Export API to get zone, unzone as well as tag based policies
from Ranger.
Signed-off-by: Pradeep <[email protected]>
---
.../apache/ranger/plugin/util/SearchFilter.java | 2 +
.../java/org/apache/ranger/biz/ServiceDBStore.java | 129 ++++++++++++++++-----
.../org/apache/ranger/common/RangerSearchUtil.java | 2 +
.../java/org/apache/ranger/common/ServiceUtil.java | 42 +++----
.../java/org/apache/ranger/rest/ServiceREST.java | 15 ++-
5 files changed, 134 insertions(+), 56 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
index 029b104..93b28a8 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
@@ -86,6 +86,8 @@ public class SearchFilter {
public static final String PLUGIN_ENTITY_TYPE =
"pluginEntityType";
public static final String PLUGIN_IP_ADDRESS =
"pluginIpAddress";
public static final String CLUSTER_NAME = "clusterName";
+ public static final String FETCH_ZONE_UNZONE_POLICIES =
"fetchZoneAndUnzonePolicies";
+ public static final String FETCH_TAG_POLICIES =
"fetchTagPolicies";
private Map<String, String> params;
private int startIndex;
diff --git
a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index fc4b40d..e1c4578 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -19,6 +19,13 @@
package org.apache.ranger.biz;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.UnknownHostException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -26,6 +33,7 @@ import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -33,13 +41,6 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.TreeSet;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.UnknownHostException;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
import javax.annotation.PostConstruct;
import javax.servlet.ServletOutputStream;
@@ -62,27 +63,12 @@ import org.apache.ranger.audit.provider.MiscUtil;
import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
import org.apache.ranger.common.AppConstants;
import org.apache.ranger.common.ContextUtil;
-import org.apache.ranger.common.MessageEnums;
-import org.apache.ranger.common.RangerCommonEnums;
-import org.apache.ranger.common.db.RangerTransactionSynchronizationAdapter;
-import org.apache.ranger.db.XXPolicyDao;
-import org.apache.ranger.entity.*;
-import org.apache.ranger.plugin.model.RangerRole;
-import org.apache.ranger.plugin.model.RangerSecurityZone;
-import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator;
-import org.apache.ranger.plugin.model.validation.RangerValidator;
-import org.apache.ranger.plugin.model.validation.ValidationFailureDetails;
-import org.apache.ranger.plugin.model.RangerPolicyDelta;
-import org.apache.ranger.plugin.policyengine.RangerPolicyEngine;
-import
org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher;
-import
org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher;
-import org.apache.ranger.plugin.service.RangerBaseService;
-import org.apache.ranger.plugin.store.ServiceStore;
-import org.apache.ranger.plugin.util.PasswordUtils;
import org.apache.ranger.common.DateUtil;
import org.apache.ranger.common.JSONUtil;
+import org.apache.ranger.common.MessageEnums;
import org.apache.ranger.common.PropertiesUtil;
import org.apache.ranger.common.RESTErrorUtil;
+import org.apache.ranger.common.RangerCommonEnums;
import org.apache.ranger.common.RangerConstants;
import org.apache.ranger.common.RangerFactory;
import org.apache.ranger.common.RangerServicePoliciesCache;
@@ -90,6 +76,7 @@ import org.apache.ranger.common.RangerVersionInfo;
import org.apache.ranger.common.SearchCriteria;
import org.apache.ranger.common.StringUtil;
import org.apache.ranger.common.UserSessionBase;
+import org.apache.ranger.common.db.RangerTransactionSynchronizationAdapter;
import org.apache.ranger.db.RangerDaoManager;
import org.apache.ranger.db.XXAccessTypeDefDao;
import org.apache.ranger.db.XXAccessTypeDefGrantsDao;
@@ -98,12 +85,39 @@ import org.apache.ranger.db.XXDataMaskTypeDefDao;
import org.apache.ranger.db.XXEnumDefDao;
import org.apache.ranger.db.XXEnumElementDefDao;
import org.apache.ranger.db.XXPolicyConditionDefDao;
+import org.apache.ranger.db.XXPolicyDao;
import org.apache.ranger.db.XXPolicyLabelMapDao;
import org.apache.ranger.db.XXResourceDefDao;
import org.apache.ranger.db.XXServiceConfigDefDao;
import org.apache.ranger.db.XXServiceConfigMapDao;
import org.apache.ranger.db.XXServiceDao;
import org.apache.ranger.db.XXServiceVersionInfoDao;
+import org.apache.ranger.entity.XXAccessTypeDef;
+import org.apache.ranger.entity.XXAccessTypeDefGrants;
+import org.apache.ranger.entity.XXContextEnricherDef;
+import org.apache.ranger.entity.XXDataHist;
+import org.apache.ranger.entity.XXDataMaskTypeDef;
+import org.apache.ranger.entity.XXEnumDef;
+import org.apache.ranger.entity.XXEnumElementDef;
+import org.apache.ranger.entity.XXGroup;
+import org.apache.ranger.entity.XXPolicy;
+import org.apache.ranger.entity.XXPolicyChangeLog;
+import org.apache.ranger.entity.XXPolicyConditionDef;
+import org.apache.ranger.entity.XXPolicyLabel;
+import org.apache.ranger.entity.XXPolicyLabelMap;
+import org.apache.ranger.entity.XXPolicyRefAccessType;
+import org.apache.ranger.entity.XXPolicyRefCondition;
+import org.apache.ranger.entity.XXPolicyRefResource;
+import org.apache.ranger.entity.XXResourceDef;
+import org.apache.ranger.entity.XXRoleRefRole;
+import org.apache.ranger.entity.XXSecurityZone;
+import org.apache.ranger.entity.XXService;
+import org.apache.ranger.entity.XXServiceConfigDef;
+import org.apache.ranger.entity.XXServiceConfigMap;
+import org.apache.ranger.entity.XXServiceDef;
+import org.apache.ranger.entity.XXServiceVersionInfo;
+import org.apache.ranger.entity.XXTrxLog;
+import org.apache.ranger.entity.XXUser;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerPolicy.RangerDataMaskPolicyItem;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem;
@@ -111,7 +125,10 @@ import
org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.model.RangerPolicy.RangerRowFilterPolicyItem;
+import org.apache.ranger.plugin.model.RangerPolicyDelta;
import org.apache.ranger.plugin.model.RangerPolicyResourceSignature;
+import org.apache.ranger.plugin.model.RangerRole;
+import org.apache.ranger.plugin.model.RangerSecurityZone;
import org.apache.ranger.plugin.model.RangerService;
import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.model.RangerServiceDef.RangerAccessTypeDef;
@@ -125,10 +142,19 @@ import
org.apache.ranger.plugin.model.RangerServiceDef.RangerResourceDef;
import org.apache.ranger.plugin.model.RangerServiceDef.RangerRowFilterDef;
import org.apache.ranger.plugin.model.RangerServiceDef.RangerServiceConfigDef;
import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper;
+import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator;
+import org.apache.ranger.plugin.model.validation.RangerValidator;
+import org.apache.ranger.plugin.model.validation.ValidationFailureDetails;
+import org.apache.ranger.plugin.policyengine.RangerPolicyEngine;
+import
org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher;
+import
org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher;
+import org.apache.ranger.plugin.service.RangerBaseService;
import org.apache.ranger.plugin.store.AbstractServiceStore;
import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil;
import org.apache.ranger.plugin.store.PList;
import org.apache.ranger.plugin.store.ServicePredicateUtil;
+import org.apache.ranger.plugin.store.ServiceStore;
+import org.apache.ranger.plugin.util.PasswordUtils;
import org.apache.ranger.plugin.util.RangerPolicyDeltaUtil;
import org.apache.ranger.plugin.util.SearchFilter;
import org.apache.ranger.plugin.util.ServicePolicies;
@@ -1810,7 +1836,6 @@ public class ServiceDBStore extends AbstractServiceStore {
MessageEnums.OPER_NO_PERMISSION);
}
}
-
return xService == null ? null :
svcService.getPopulatedViewObject(xService);
}
@@ -2155,14 +2180,60 @@ public class ServiceDBStore extends
AbstractServiceStore {
if(LOG.isDebugEnabled()) {
LOG.debug("==> ServiceDBStore.getPolicies()");
}
+ Boolean fetchTagPolicies =
Boolean.valueOf(filter.getParam(SearchFilter.FETCH_TAG_POLICIES));
+ Boolean fetchAllZonePolicies =
Boolean.valueOf(filter.getParam(SearchFilter.FETCH_ZONE_UNZONE_POLICIES));
+ String zoneName =
filter.getParam(SearchFilter.ZONE_NAME);
+
+ List<RangerPolicy> ret = new ArrayList<RangerPolicy>();
RangerPolicyList policyList = searchRangerPolicies(filter);
- List<RangerPolicy> ret = policyList.getPolicies();
+ List<RangerPolicy> resourcePolicies = policyList.getPolicies();
+ List<RangerPolicy> tagPolicies = new ArrayList<RangerPolicy>();
+
+ if(fetchTagPolicies) {
+ tagPolicies =
searchRangerTagPoliciesOnBasisOfServiceName(resourcePolicies);
+ Iterator<RangerPolicy> itr = tagPolicies.iterator();
+ while (itr.hasNext()) {
+ RangerPolicy pol = (RangerPolicy) itr.next();
+ if(!fetchAllZonePolicies) {
+ if(StringUtils.isNotEmpty(zoneName)) {
+
if(!zoneName.equals(pol.getZoneName())){
+ itr.remove();
+ }
+ } else {
+
if(StringUtils.isNotEmpty(pol.getZoneName())) {
+ itr.remove();
+ }
+ }
+ }
+ }
+ }
if(LOG.isDebugEnabled()) {
LOG.debug("<== ServiceDBStore.getPolicies()");
}
+ ret.addAll(resourcePolicies);
+ ret.addAll(tagPolicies);
return ret;
}
+ private List<RangerPolicy>
searchRangerTagPoliciesOnBasisOfServiceName(List<RangerPolicy>
allExceptTagPolicies) throws Exception {
+ Set<String> rangerServiceNames = new HashSet<String>();
+ for(RangerPolicy pol : allExceptTagPolicies) {
+ rangerServiceNames.add(pol.getService());
+ }
+ List<RangerPolicy> retPolicies = new ArrayList<RangerPolicy>();
+ for(String eachRangerService : rangerServiceNames) {
+ List<RangerPolicy> policies = new
ArrayList<RangerPolicy>();
+ RangerService rangerServiceObj =
getServiceByName(eachRangerService);
+ RangerService rangerTagService =
getServiceByName(rangerServiceObj.getTagService());
+ if(rangerTagService != null) {
+ ServicePolicies servicePolicies =
RangerServicePoliciesCache.getInstance().getServicePolicies(rangerTagService.getName(),rangerTagService.getId(),
-1L, true, this);
+ policies = servicePolicies != null ?
servicePolicies.getPolicies() : null;
+ retPolicies.addAll(policies);
+ }
+ }
+ return retPolicies;
+ }
+
@Override
public Long getPolicyId(final Long serviceId, final String policyName,
final Long zoneId) {
if(LOG.isDebugEnabled()) {
@@ -2264,8 +2335,10 @@ public class ServiceDBStore extends AbstractServiceStore
{
}
List<RangerPolicy> ret = getServicePolicies(service, filter);
- if(StringUtils.isBlank(zoneName)) {
- ret = noZoneFilter(ret);
+
if(!"true".equalsIgnoreCase(filter.getParam(SearchFilter.FETCH_ZONE_UNZONE_POLICIES)))
{
+ if(StringUtils.isBlank(zoneName)) {
+ ret = noZoneFilter(ret);
+ }
}
if(LOG.isDebugEnabled()) {
LOG.debug("<== ServiceDBStore.getServicePolicies(" +
serviceId + ") : policy-count=" + (ret == null ? 0 : ret.size()));
diff --git
a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
index 954144b..99af818 100644
---
a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
+++
b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
@@ -81,6 +81,8 @@ public class RangerSearchUtil extends SearchUtil {
ret.setParam(SearchFilter.GROUP_NAME_PARTIAL,
request.getParameter(SearchFilter.GROUP_NAME_PARTIAL));
ret.setParam(SearchFilter.USER_NAME_PARTIAL,
request.getParameter(SearchFilter.USER_NAME_PARTIAL));
ret.setParam(SearchFilter.CLUSTER_NAME,
request.getParameter(SearchFilter.CLUSTER_NAME));
+ ret.setParam(SearchFilter.FETCH_ZONE_UNZONE_POLICIES,
request.getParameter(SearchFilter.FETCH_ZONE_UNZONE_POLICIES));
+ ret.setParam(SearchFilter.FETCH_TAG_POLICIES,
request.getParameter(SearchFilter.FETCH_TAG_POLICIES));
for (Map.Entry<String, String[]> e :
request.getParameterMap().entrySet()) {
String name = e.getKey();
String[] values = e.getValue();
diff --git
a/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java
b/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java
index 7c44e6c..2b1a3fa 100644
--- a/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java
+++ b/security-admin/src/main/java/org/apache/ranger/common/ServiceUtil.java
@@ -1578,6 +1578,7 @@ public class ServiceUtil {
public List<RangerPolicy>
getMatchingPoliciesForResource(HttpServletRequest request,
List<RangerPolicy> policyLists) {
List<RangerPolicy> policies = new ArrayList<RangerPolicy>();
+ final String serviceTypeForTag =
EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME;
if (request != null) {
String resource =
request.getParameter(SearchFilter.POL_RESOURCE);
String serviceType =
request.getParameter(SearchFilter.SERVICE_TYPE);
@@ -1587,32 +1588,25 @@ public class ServiceUtil {
RangerPolicy.RangerPolicyResource
rangerPolicyResource = null;
for (RangerPolicy rangerPolicy : policyLists) {
if (rangerPolicy != null) {
- rangerPolicyResourceMap =
rangerPolicy.getResources();
- if (rangerPolicyResourceMap !=
null) {
- if
(rangerPolicyResourceMap.containsKey("path")) {
-
rangerPolicyResource = rangerPolicyResourceMap.get("path");
- if
(rangerPolicyResource != null) {
-
resourceList = rangerPolicyResource.getValues();
- if
(CollectionUtils.isNotEmpty(resourceList) && resourceList.size() == 1) {
-
String resourcePath = resourceList.get(0);
-
if (!StringUtil.isEmpty(resourcePath)) {
-
if (resourcePath.equals(resource)
-
|| resourcePath.startsWith(resource + "/")) {
-
policies.add(rangerPolicy);
-
}
+
if(serviceTypeForTag.equals(rangerPolicy.getServiceType())) {
+
policies.add(rangerPolicy);
+ }else {
+ rangerPolicyResourceMap
= rangerPolicy.getResources();
+ if
(rangerPolicyResourceMap != null) {
+ if
(rangerPolicyResourceMap.containsKey("path")) {
+
rangerPolicyResource = rangerPolicyResourceMap.get("path");
+ if
(rangerPolicyResource != null) {
+
resourceList = rangerPolicyResource.getValues();
+
if (CollectionUtils.isNotEmpty(resourceList) &&
resourceList.contains(resource)) {
+
policies.add(rangerPolicy);
}
}
- }
- } else if
(rangerPolicyResourceMap.containsKey("database")) {
-
rangerPolicyResource = rangerPolicyResourceMap.get("database");
- if
(rangerPolicyResource != null) {
-
resourceList = rangerPolicyResource.getValues();
- if
(CollectionUtils.isNotEmpty(resourceList) && resourceList.size() == 1) {
-
String resourcePath = resourceList.get(0);
-
if (!StringUtil.isEmpty(resourcePath)) {
-
if (resourcePath.equals(resource)) {
-
policies.add(rangerPolicy);
-
}
+ } else if
(rangerPolicyResourceMap.containsKey("database")) {
+
rangerPolicyResource = rangerPolicyResourceMap.get("database");
+ if
(rangerPolicyResource != null) {
+
resourceList = rangerPolicyResource.getValues();
+
if (CollectionUtils.isNotEmpty(resourceList) &&
resourceList.contains(resource)) {
+
policies.add(rangerPolicy);
}
}
}
diff --git
a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
index 3d44315..8ee181a 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
@@ -2050,7 +2050,7 @@ public class ServiceREST {
RangerPerfTracer perf = null;
SearchFilter filter =
searchUtil.getSearchFilter(request,policyService.sortFields);
- String zoneName = filter.getParam("zoneName");
+ requestParamsValidation(filter);
try {
if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf =
RangerPerfTracer.getPerfTracer(PERF_LOG,"ServiceREST.getPoliciesInJson()");
@@ -2062,9 +2062,7 @@ public class ServiceREST {
List<RangerPolicy> policyLists = new
ArrayList<RangerPolicy>();
policyLists = getAllFilteredPolicyList(filter, request,
policyLists);
- if (StringUtils.isBlank(zoneName)) {// if zoneName not
provided in search parameter, return only policies which are not in any zone.
- policyLists =
svcStore.noZoneFilter(policyLists);
- }
+
if (CollectionUtils.isNotEmpty(policyLists)) {
for (RangerPolicy rangerPolicy : policyLists) {
if (rangerPolicy != null) {
@@ -2102,6 +2100,15 @@ public class ServiceREST {
}
}
+ private void requestParamsValidation(SearchFilter filter) {
+ Boolean fetchAllZonePolicies =
Boolean.valueOf(filter.getParam(SearchFilter.FETCH_ZONE_UNZONE_POLICIES));
+ String zoneName =
filter.getParam(SearchFilter.ZONE_NAME);
+
+ if (fetchAllZonePolicies && StringUtils.isNotEmpty(zoneName)) {
+ throw restErrorUtil.createRESTException("Invalid parameter:
" + SearchFilter.ZONE_NAME + " can not be provided, along with " +
SearchFilter.FETCH_ZONE_UNZONE_POLICIES + "=true");
+ }
+ }
+
@POST
@Path("/policies/importPoliciesFromFile")
@Consumes({MediaType.MULTIPART_FORM_DATA, MediaType.APPLICATION_JSON})
