This is an automated email from the ASF dual-hosted git repository.
pradeep pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new e01b34502 RANGER-4996: Made meta attributes fields in policy rest api
response configurable for retrieval
e01b34502 is described below
commit e01b345023870ca52d373d19534c3d1c984668f2
Author: Guru Thejus Arveti <[email protected]>
AuthorDate: Wed Nov 13 22:15:37 2024 +0530
RANGER-4996: Made meta attributes fields in policy rest api response
configurable for retrieval
Signed-off-by: Pradeep Agrawal <[email protected]>
Change-Id: I25357987d5b8fa93a29fe8a6cdfe916318991ee4
---
.../apache/ranger/plugin/store/ServiceStore.java | 2 ++
.../java/org/apache/ranger/biz/ServiceDBStore.java | 34 ++++++++++++++++++++--
.../ranger/common/RangerServicePoliciesCache.java | 3 +-
.../java/org/apache/ranger/db/XXPolicyDao.java | 12 ++++++++
.../java/org/apache/ranger/rest/PublicAPIsv2.java | 5 +++-
.../java/org/apache/ranger/rest/ServiceREST.java | 4 +++
.../main/resources/META-INF/jpa_named_queries.xml | 4 +++
7 files changed, 59 insertions(+), 5 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
index 97072391e..bf0f95d34 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
@@ -124,4 +124,6 @@ public interface ServiceStore {
long getPoliciesCount(final String serviceName);
Map<String, String> getServiceConfigForPlugin(Long serviceId);
+
+ List<RangerPolicy> getPoliciesWithMetaAttributes(List<RangerPolicy>
policies);
}
diff --git
a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index 0dee87734..a21b8c8d2 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -27,6 +27,7 @@ import java.net.UnknownHostException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -42,6 +43,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.TreeSet;
+import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import javax.servlet.ServletOutputStream;
@@ -50,6 +52,7 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.SerializationUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
@@ -296,7 +299,7 @@ public class ServiceDBStore extends AbstractServiceStore {
RangerPolicyService policyService;
@Autowired
- RangerPolicyLabelsService<XXPolicyLabel, ?> policyLabelsService;
+ RangerPolicyLabelsService<XXPolicyLabel, ?> policyLabelsService;
@Autowired
XUserService xUserService;
@@ -371,7 +374,6 @@ public class ServiceDBStore extends AbstractServiceStore {
private ServicePredicateUtil predicateUtil = null;
private RangerAdminConfig config = null;
-
@Override
public void init() throws Exception {
if (LOG.isDebugEnabled()) {
@@ -2674,6 +2676,34 @@ public class ServiceDBStore extends AbstractServiceStore
{
return ret;
}
+ @Override
+ public List<RangerPolicy>
getPoliciesWithMetaAttributes(List<RangerPolicy> policiesList) {
+ if (CollectionUtils.isNotEmpty(policiesList)) {
+ List<RangerPolicy> policies = new ArrayList<>();
+ for (RangerPolicy policy : policiesList) {
+ RangerPolicy policyCopy = (RangerPolicy)
SerializationUtils.clone(policy);
+ policies.add(policyCopy);
+ }
+
+ List<Object[]> policytimeMetaDataList =
daoMgr.getXXPolicy().getMetaAttributesForPolicies(policies.stream().map(RangerPolicy::getId).collect(Collectors.toList()));
+ if (CollectionUtils.isNotEmpty(policytimeMetaDataList))
{
+ Map<Long, List<Date>> policyMap =
policytimeMetaDataList.stream()
+ .filter(row -> row != null &&
row.length == 3 && row[0] != null && row[1] != null && row[2] != null)
+ .collect(Collectors.toMap(row
-> (Long) row[0], row-> Arrays.asList((Date) row[1], (Date) row[2])));
+
+ for (RangerPolicy policy : policies) {
+ List<Date> timeMetaData =
policyMap.get(policy.getId());
+ if (timeMetaData != null &&
timeMetaData.size() == 2) {
+
policy.setCreateTime(timeMetaData.get(0));
+
policy.setUpdateTime(timeMetaData.get(1));
+ }
+ }
+ }
+ return policies;
+ }
+ return policiesList;
+ }
+
private List<RangerPolicy> getServicePolicies(XXService service,
SearchFilter filter) throws Exception {
if(LOG.isDebugEnabled()) {
LOG.debug("==> ServiceDBStore.getServicePolicies()");
diff --git
a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
index c8d1e5d08..df27611a1 100644
---
a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
+++
b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java
@@ -453,14 +453,12 @@ public class RangerServicePoliciesCache {
LOG.debug("Initializing
ServicePolicies cache for the first time");
}
servicePolicies =
servicePoliciesFromDb;
- pruneUnusedAttributes();
} else if
(servicePoliciesFromDb.getPolicyDeltas() == null) {
// service-policies are loaded
because service/service-def changed
if (LOG.isDebugEnabled()) {
LOG.debug("Complete set
of policies are loaded from database, because of some disqualifying event");
}
servicePolicies =
servicePoliciesFromDb;
- pruneUnusedAttributes();
isCacheReloadedByDQEvent = true;
} else { // Previously cached service
policies are still valid - no service/service-def change
// Rebuild policies cache from
original policies and deltas
@@ -497,6 +495,7 @@ public class RangerServicePoliciesCache {
}
}
}
+ pruneUnusedAttributes();
this.deltaCache = null;
} else {
LOG.error("Could not get policies from
database, from-version:[" + cachedServicePoliciesVersion + ")");
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java
b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java
index c65b961fc..48ca58c08 100755
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import javax.persistence.NoResultException;
+import javax.persistence.Query;
import org.apache.commons.collections.ListUtils;
import org.apache.commons.lang.StringUtils;
@@ -452,6 +453,17 @@ public class XXPolicyDao extends BaseDao<XXPolicy> {
return ret;
}
+ public List<Object[]> getMetaAttributesForPolicies(List<Long>
policyIds) {
+ if (policyIds == null || policyIds.isEmpty()) {
+ return Collections.emptyList();
+ }
+
+ Query query =
getEntityManager().createNamedQuery("XXPolicy.getMetaAttributesForPolicies",
tClass);
+ query.setParameter("policyIds", policyIds);
+
+ return query.getResultList();
+ }
+
public List<XXPolicy> getProjectPolicies(Long projectId) {
List<XXPolicy> ret = Collections.emptyList();
diff --git
a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java
b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java
index b93b63f85..d9a214099 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java
@@ -549,10 +549,13 @@ public class PublicAPIsv2 {
ret = serviceREST.getPolicies(request).getPolicies();
+ boolean includeMetaAttributes =
Boolean.parseBoolean(request.getParameter("includeMetaAttributes"));
+ if (includeMetaAttributes) {
+ ret = serviceREST.getPoliciesWithMetaAttributes(ret);
+ }
if(logger.isDebugEnabled()) {
logger.debug("<== PublicAPIsv2.getPolicies(Request: " +
request.getQueryString() + " Result Size: " + ret.size() );
}
-
return ret;
}
diff --git
a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
index ce3a95766..9d2730937 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java
@@ -3773,6 +3773,10 @@ public class ServiceREST {
return
RangerPolicyAdminCacheForEngineOptions.getInstance().getServicePoliciesAdmin(serviceName,
svcStore, zoneStore,roleDBStore, defaultAdminOptions);
}
+ public List<RangerPolicy>
getPoliciesWithMetaAttributes(List<RangerPolicy> policies) {
+ return svcStore.getPoliciesWithMetaAttributes(policies);
+ }
+
@GET
@Path("/checksso")
@Produces(MediaType.TEXT_PLAIN)
diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
index 1a2acd188..855e27ae5 100755
--- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
+++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
@@ -406,6 +406,10 @@
<query>select obj from XXPolicy obj where obj.name = :polName
and obj.service = :serviceId and obj.zoneId = :zoneId order by obj.id</query>
</named-query>
+ <named-query name="XXPolicy.getMetaAttributesForPolicies">
+ <query>SELECT obj.id, obj.createTime, obj.updateTime FROM
XXPolicy obj WHERE obj.id IN :policyIds</query>
+ </named-query>
+
<named-query name="XXPolicy.findByServiceId">
<query>select obj from XXPolicy obj where obj.service =
:serviceId order by obj.id</query>
</named-query>
