This is an automated email from the ASF dual-hosted git repository.
rmani pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new c334ba80a4 RANGER-5111: Enhance Datashare Summary API to Include
Labels, Keywords, and Share Counts for Actionable Filters
c334ba80a4 is described below
commit c334ba80a4bb38653a00f61cd1bec0927194a018
Author: Radhika Kundam <[email protected]>
AuthorDate: Wed Jan 22 16:53:35 2025 -0800
RANGER-5111: Enhance Datashare Summary API to Include Labels, Keywords, and
Share Counts for Actionable Filters
Signed-off-by: Ramesh Mani <[email protected]>
---
.../org/apache/ranger/plugin/model/RangerGds.java | 51 ++++++++++++--
.../java/org/apache/ranger/plugin/store/PList.java | 22 +++++++
.../java/org/apache/ranger/biz/GdsDBStore.java | 77 ++++++++++++++++++++--
.../main/java/org/apache/ranger/rest/GdsREST.java | 39 +++++++++--
4 files changed, 173 insertions(+), 16 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerGds.java
b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerGds.java
index 260ebc0a8b..6bdc02691a 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerGds.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerGds.java
@@ -26,11 +26,13 @@
import
org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.model.RangerPrincipal.PrincipalType;
+import org.apache.ranger.plugin.store.PList;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -167,8 +169,8 @@ public StringBuilder toString(StringBuilder sb) {
.append("acl={").append(acl).append("} ")
.append("validitySchedule={").append(validitySchedule).append("} ")
.append("termsOfUse={").append(termsOfUse).append("} ")
- .append("labels={").append(validitySchedule).append("} ")
- .append("keywords={").append(termsOfUse).append("} ")
+ .append("labels={").append(labels).append("} ")
+ .append("keywords={").append(keywords).append("} ")
.append("}");
return sb;
@@ -860,9 +862,48 @@ public StringBuilder toString(StringBuilder sb) {
.append("aclPrincipalsCount={").append(aclPrincipalsCount).append("} ")
.append("totalResourceCount={").append(totalResourceCount).append("} ")
.append("dataShares={").append(dataShares).append("} ")
-
.append("validitySchedule={").append(totalResourceCount).append("} ")
- .append("labels={").append(totalResourceCount).append("} ")
- .append("keywords={").append(totalResourceCount).append("}
")
+
.append("validitySchedule={").append(validitySchedule).append("} ")
+ .append("labels={").append(labels).append("} ")
+ .append("keywords={").append(keywords).append("} ")
+ .append("}");
+
+ return sb;
+ }
+ }
+
+ @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY)
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class DatasetsSummary extends PList<DatasetSummary> {
+ private static final long serialVersionUID = 1L;
+ private Map<String, Map<String, Integer>> additionalInfo;
+
+ public DatasetsSummary() {
+ super();
+ }
+
+ public DatasetsSummary(PList<DatasetSummary> datasetSummary,
Map<String, Map<String, Integer>> additionalInfo) {
+ super(datasetSummary);
+ this.additionalInfo = (additionalInfo != null) ? additionalInfo :
Collections.emptyMap();
+ }
+
+ public Map<String, Map<String, Integer>> getAdditionalInfo() {
+ return additionalInfo;
+ }
+
+ public void setAdditionalInfo(Map<String, Map<String, Integer>>
additionalInfo) {
+ this.additionalInfo = additionalInfo;
+ }
+
+ @Override
+ public String toString() {
+ return toString(new StringBuilder()).toString();
+ }
+
+ public StringBuilder toString(StringBuilder sb) {
+ sb.append("DatasetsSummary={")
+ .append("list={").append(this.list).append("} ")
+
.append("additionalInfo={").append(additionalInfo).append("} ")
.append("}");
return sb;
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java
b/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java
index 4bf2ee02fe..1a1b16a5bf 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java
@@ -19,6 +19,8 @@
package org.apache.ranger.plugin.store;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
public class PList<T> implements java.io.Serializable {
@@ -75,6 +77,26 @@ public PList(List<T> list, int startIndex, int pageSize,
long totalCount, int re
this.sortBy = sortBy;
}
+ public PList(PList<T> other) {
+ if (other != null) {
+ this.list = other.getList() != null ? new
ArrayList<>(other.getList()) : Collections.emptyList();
+ this.startIndex = other.getStartIndex();
+ this.pageSize = other.getPageSize();
+ this.totalCount = other.getTotalCount();
+ this.resultSize = other.getResultSize();
+ this.sortType = other.getSortType();
+ this.sortBy = other.getSortBy();
+ } else {
+ this.list = Collections.emptyList();
+ this.startIndex = 0;
+ this.pageSize = 0;
+ this.totalCount = 0;
+ this.resultSize = 0;
+ this.sortType = null;
+ this.sortBy = null;
+ }
+ }
+
public int getListSize() {
return list == null ? 0 : list.size();
}
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
b/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
index 7916f08188..312a67a28f 100755
--- a/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
@@ -48,6 +48,7 @@
import org.apache.ranger.plugin.model.RangerGds.DataShareInDatasetSummary;
import org.apache.ranger.plugin.model.RangerGds.DataShareSummary;
import org.apache.ranger.plugin.model.RangerGds.DatasetSummary;
+import org.apache.ranger.plugin.model.RangerGds.DatasetsSummary;
import org.apache.ranger.plugin.model.RangerGds.GdsPermission;
import org.apache.ranger.plugin.model.RangerGds.GdsShareStatus;
import org.apache.ranger.plugin.model.RangerGds.RangerDataShare;
@@ -120,6 +121,9 @@ public class GdsDBStore extends AbstractGdsStore {
public static final String NOT_AUTHORIZED_TO_VIEW_PROJECT_POLICIES = "User
is not authorized to view policies for this dataset";
public static final String GDS_POLICY_NAME_TIMESTAMP_SEP = "@";
+ public static final String LABELS =
"labelCounts";
+ public static final String KEYWORDS =
"keywordCounts";
+
private static final Set<Integer> SHARE_STATUS_AGR = new
HashSet<>(Arrays.asList(GdsShareStatus.ACTIVE.ordinal(),
GdsShareStatus.GRANTED.ordinal(), GdsShareStatus.REQUESTED.ordinal()));
@Autowired
@@ -1370,19 +1374,68 @@ public ServiceGdsInfo getGdsInfoIfUpdated(String
serviceName, Long lastKnownVers
}
public PList<DatasetSummary> getDatasetSummary(SearchFilter filter) {
- LOG.debug("==> getDatasetSummary({})", filter);
+ return getDatasetSummary(filter, false);
+ }
- PList<RangerDataset> datasets = getUnscrubbedDatasets(filter);
- List<DatasetSummary> datasetSummary =
toDatasetSummary(datasets.getList(), getGdsPermissionFromFilter(filter));
- PList<DatasetSummary> ret = new PList<>(datasetSummary,
datasets.getStartIndex(), datasets.getPageSize(), datasets.getTotalCount(),
datasets.getResultSize(), datasets.getSortType(), datasets.getSortBy());
+ public DatasetsSummary getEnhancedDatasetSummary(SearchFilter filter) {
+ return getDatasetSummary(filter, true);
+ }
- ret.setQueryTimeMS(datasets.getQueryTimeMS());
+ public DatasetsSummary getDatasetSummary(SearchFilter filter, boolean
includeAdditionalInfo) {
+ LOG.debug("==> getDatasetSummary({}, {})", filter,
includeAdditionalInfo);
- LOG.debug("<== getDatasetSummary({}): ret={}", filter, ret);
+ PList<RangerDataset> datasets;
+ Map<String, Map<String, Integer>> additionalInfo = null;
+
+ if (includeAdditionalInfo) {
+ List<RangerDataset> datasetsMatchingCriteria =
fetchDatasetsBySearchCriteria(filter);
+ additionalInfo =
buildAdditionalInfoForDatasets(datasetsMatchingCriteria);
+ datasets =
applyPaginataionAndSorting(datasetsMatchingCriteria, filter);
+ } else {
+ datasets = getUnscrubbedDatasets(filter);
+ }
+
+ List<DatasetSummary> datasetSummary =
toDatasetSummary(datasets.getList(), getGdsPermissionFromFilter(filter));
+ PList<DatasetSummary> paginatedDatasetSummary =
createdPaginatedDatasetSummary(datasets, datasetSummary);
+ DatasetsSummary ret = new
DatasetsSummary(paginatedDatasetSummary, additionalInfo);
+
+ LOG.debug("<== getDatasetSummary({}, {}): ret={}", filter,
includeAdditionalInfo, ret);
return ret;
}
+ private Map<String, Map<String, Integer>>
buildAdditionalInfoForDatasets(List<RangerDataset> datasets) {
+ Map<String, Map<String, Integer>> additionalInfo = new HashMap<>();
+ for (RangerDataset dataset : datasets) {
+ updateAdditionalInfo(LABELS, dataset.getLabels(), additionalInfo);
+ updateAdditionalInfo(KEYWORDS, dataset.getKeywords(),
additionalInfo);
+ }
+ return additionalInfo;
+ }
+
+ private void updateAdditionalInfo(String field, List<String> fieldValues,
Map<String, Map<String, Integer>> additionalInfo) {
+ if (CollectionUtils.isNotEmpty(fieldValues)) {
+ Map<String, Integer> aggregatedFieldMap =
additionalInfo.computeIfAbsent(field, key -> new HashMap<>());
+ for (String value : fieldValues) {
+ aggregatedFieldMap.put(value,
aggregatedFieldMap.getOrDefault(value, 0) + 1);
+ }
+ }
+ }
+
+ private PList<DatasetSummary>
createdPaginatedDatasetSummary(PList<RangerDataset> datasets,
List<DatasetSummary> datasetSummary) {
+ PList<DatasetSummary> paginatedDatasetSummary = new PList<>(
+ datasetSummary,
+ datasets.getStartIndex(),
+ datasets.getPageSize(),
+ datasets.getTotalCount(),
+ datasets.getResultSize(),
+ datasets.getSortType(),
+ datasets.getSortBy());
+
+ paginatedDatasetSummary.setQueryTimeMS(datasets.getQueryTimeMS());
+ return paginatedDatasetSummary;
+ }
+
public PList<DataShareSummary> getDataShareSummary(SearchFilter filter) {
LOG.debug("==> getDataShareSummary({})", filter);
@@ -1698,6 +1751,12 @@ private PList<RangerProject>
getUnscrubbedProjects(SearchFilter filter) {
}
private PList<RangerDataset> getUnscrubbedDatasets(SearchFilter filter) {
+ List<RangerDataset> datasets = fetchDatasetsBySearchCriteria(filter);
+
+ return applyPaginataionAndSorting(datasets, filter);
+ }
+
+ private List<RangerDataset> fetchDatasetsBySearchCriteria(SearchFilter
filter) {
filter.setParam(SearchFilter.RETRIEVE_ALL_PAGES, "true");
GdsPermission gdsPermission =
getGdsPermissionFromFilter(filter);
@@ -1727,10 +1786,14 @@ private PList<RangerDataset>
getUnscrubbedDatasets(SearchFilter filter) {
}
}
+ return datasets;
+ }
+
+ private PList<RangerDataset>
applyPaginataionAndSorting(List<RangerDataset> datasets, SearchFilter filter) {
int maxRows = filter.getMaxRows();
int startIndex = filter.getStartIndex();
- return getPList(datasets, startIndex, maxRows, result.getSortBy(),
result.getSortType());
+ return getPList(datasets, startIndex, maxRows, filter.getSortBy(),
filter.getSortType());
}
private PList<RangerDataShare> getUnscrubbedDataShares(SearchFilter
filter) {
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/GdsREST.java
b/security-admin/src/main/java/org/apache/ranger/rest/GdsREST.java
index 0d3ef3d767..05bc4fd17b 100755
--- a/security-admin/src/main/java/org/apache/ranger/rest/GdsREST.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/GdsREST.java
@@ -36,6 +36,7 @@
import org.apache.ranger.plugin.model.RangerGds.DataShareInDatasetSummary;
import org.apache.ranger.plugin.model.RangerGds.DataShareSummary;
import org.apache.ranger.plugin.model.RangerGds.DatasetSummary;
+import org.apache.ranger.plugin.model.RangerGds.DatasetsSummary;
import org.apache.ranger.plugin.model.RangerGds.RangerDataShare;
import org.apache.ranger.plugin.model.RangerGds.RangerDataShareInDataset;
import org.apache.ranger.plugin.model.RangerGds.RangerDataset;
@@ -449,13 +450,13 @@ public PList<DatasetSummary> getDatasetSummary(@Context
HttpServletRequest reque
LOG.debug("==> GdsREST.getDatasetSummary()");
PList<DatasetSummary> ret;
- RangerPerfTracer perf =
RangerPerfTracer.getPerfTracer(PERF_LOG, "GdsREST.getDatasetSummary()");
- SearchFilter filter = null;
+ RangerPerfTracer perf = RangerPerfTracer.getPerfTracer(PERF_LOG,
"GdsREST.getDatasetSummary()");
+ SearchFilter filter = null;
try {
filter = searchUtil.getSearchFilter(request,
datasetService.sortFields);
- ret = gdsStore.getDatasetSummary(filter);
+ ret = gdsStore.getDatasetSummary(filter);
} catch (WebApplicationException we) {
throw we;
} catch (Throwable ex) {
@@ -466,7 +467,37 @@ public PList<DatasetSummary> getDatasetSummary(@Context
HttpServletRequest reque
RangerPerfTracer.log(perf);
}
- LOG.debug("<== GdsREST.getDatasetSummary(): {}", ret);
+ LOG.debug("<== GdsREST.getDatasetSummary()");
+
+ return ret;
+ }
+
+ @GET
+ @Path("/dataset/enhancedsummary")
+ @Produces("application/json")
+ @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" +
RangerAPIList.GET_DATASET_SUMMARY + "\")")
+ public DatasetsSummary getEnhancedDatasetSummary(@Context
HttpServletRequest request) {
+ LOG.debug("==> GdsREST.getEnhancedDatasetSummary()");
+
+ DatasetsSummary ret;
+ RangerPerfTracer perf = RangerPerfTracer.getPerfTracer(PERF_LOG,
"GdsREST.getEnhancedDatasetSummary()");
+ SearchFilter filter = null;
+
+ try {
+ filter = searchUtil.getSearchFilter(request,
datasetService.sortFields);
+
+ ret = gdsStore.getEnhancedDatasetSummary(filter);
+ } catch (WebApplicationException we) {
+ throw we;
+ } catch (Throwable ex) {
+ LOG.error("getEnhancedDatasetSummary({}) failed", filter, ex);
+
+ throw restErrorUtil.createRESTException(ex.getMessage());
+ } finally {
+ RangerPerfTracer.log(perf);
+ }
+
+ LOG.debug("<== GdsREST.getEnhancedDatasetSummary()");
return ret;
}
