This is an automated email from the ASF dual-hosted git repository.
rmani pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 8c543062f5 RANGER-5129: Enhance Dataset Filtering by Validity Schedule
and Expiration Date Range
8c543062f5 is described below
commit 8c543062f50d11c453a8dccfb177a98b4f0d42ff
Author: Radhika Kundam <[email protected]>
AuthorDate: Mon Feb 10 14:03:55 2025 -0800
RANGER-5129: Enhance Dataset Filtering by Validity Schedule and Expiration
Date Range
Signed-off-by: Ramesh Mani <[email protected]>
---
.../RangerValidityScheduleEvaluator.java | 2 +-
.../apache/ranger/plugin/util/SearchFilter.java | 4 +
.../java/org/apache/ranger/biz/GdsDBStore.java | 70 ++++++++++
.../org/apache/ranger/common/RangerSearchUtil.java | 3 +
.../java/org/apache/ranger/rest/TestGdsREST.java | 149 ++++++++++++++++++++-
5 files changed, 226 insertions(+), 2 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerValidityScheduleEvaluator.java
b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerValidityScheduleEvaluator.java
index 027c2140c6..5dac4d9f93 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerValidityScheduleEvaluator.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerValidityScheduleEvaluator.java
@@ -50,7 +50,7 @@ public class RangerValidityScheduleEvaluator {
private static final TimeZone defaultTZ = TimeZone.getDefault();
- private static final ThreadLocal<DateFormat> DATE_FORMATTER =
ThreadLocal.withInitial(() -> new
SimpleDateFormat(RangerValiditySchedule.VALIDITY_SCHEDULE_DATE_STRING_SPECIFICATION));
+ public static final ThreadLocal<DateFormat> DATE_FORMATTER =
ThreadLocal.withInitial(() -> new
SimpleDateFormat(RangerValiditySchedule.VALIDITY_SCHEDULE_DATE_STRING_SPECIFICATION));
private final Date startTime;
private final Date endTime;
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
index a260a87828..5a5556a2f2 100755
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
@@ -140,6 +140,10 @@ public class SearchFilter {
public static final String IS_DISTINCT = "isDistinct";
// search, sort
public static final String RETRIEVE_ALL_PAGES =
"retrieveAllPages"; // search
public static final String SHARED_WITH_ME = "sharedWithMe";
// search
+ public static final String VALIDITY_EXPIRY_START =
"validityExpiryStart"; // search
+ public static final String VALIDITY_EXPIRY_END =
"validityExpiryEnd"; // search
+ public static final String VALIDITY_TIME_ZONE = "timeZone";
+ public static final String DEFAULT_TIME_ZONE = "GMT";
private Map<String, String> params;
private Map<String, Object[]> multiValueParams;
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
b/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
index 312a67a28f..b1431cf71d 100755
--- a/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
@@ -64,6 +64,10 @@
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.model.RangerPolicyDelta;
import org.apache.ranger.plugin.model.RangerPrincipal.PrincipalType;
+import org.apache.ranger.plugin.model.RangerValiditySchedule;
+import
org.apache.ranger.plugin.model.validation.RangerValidityScheduleValidator;
+import org.apache.ranger.plugin.model.validation.ValidationFailureDetails;
+import
org.apache.ranger.plugin.policyevaluator.RangerValidityScheduleEvaluator;
import org.apache.ranger.plugin.store.AbstractGdsStore;
import org.apache.ranger.plugin.store.PList;
import org.apache.ranger.plugin.store.ServiceStore;
@@ -91,10 +95,12 @@
import javax.annotation.PostConstruct;
+import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
@@ -107,6 +113,7 @@
import java.util.stream.Collectors;
import static
org.apache.ranger.db.XXGlobalStateDao.RANGER_GLOBAL_STATE_NAME_GDS;
+import static
org.apache.ranger.plugin.policyevaluator.RangerValidityScheduleEvaluator.DATE_FORMATTER;
import static
org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_GDS_NAME;
@Component
@@ -1786,9 +1793,72 @@ private List<RangerDataset>
fetchDatasetsBySearchCriteria(SearchFilter filter) {
}
}
+ filterDatasetsByValidityExpiration(filter, datasets);
+
return datasets;
}
+ public void filterDatasetsByValidityExpiration(SearchFilter filter,
List<RangerDataset> datasets) {
+ LOG.debug("==> filterDatasetsByValidityExpiration({}, {})", filter,
datasets);
+ String validityCheckStart =
filter.getParam(SearchFilter.VALIDITY_EXPIRY_START);
+ String validityCheckEnd =
filter.getParam(SearchFilter.VALIDITY_EXPIRY_END);
+ String validityTimeZone =
filter.getParam(SearchFilter.VALIDITY_TIME_ZONE);
+ RangerValiditySchedule validityCheckFilter = new
RangerValiditySchedule(validityCheckStart, validityCheckEnd, validityTimeZone,
null);
+ List<ValidationFailureDetails> failures = new
ArrayList<>();
+ RangerValiditySchedule validatedValidityCheckFilter =
validateValidityCheckFilter(validityCheckFilter, failures);
+
+ if (validatedValidityCheckFilter != null) {
+ RangerValidityScheduleEvaluator validityScheduleEvaluator = new
RangerValidityScheduleEvaluator(validatedValidityCheckFilter);
+ datasets.removeIf(dataset -> !isDatasetExpiring(dataset,
validityScheduleEvaluator, failures));
+ }
+
+ if (CollectionUtils.isNotEmpty(failures)) {
+ throw restErrorUtil.createRESTException("Error in finding datasets
expiring between '" + validityCheckStart + "' and '" + validityCheckEnd + "': "
+ failures);
+ }
+ LOG.debug("==> filterDatasetsByValidityExpiration({}, {})", filter,
datasets);
+ }
+
+ private boolean isDatasetExpiring(RangerDataset dataset,
RangerValidityScheduleEvaluator validityScheduleEvaluator,
List<ValidationFailureDetails> failures) {
+ if (dataset.getValiditySchedule() == null) {
+ return false;
+ }
+ String datasetValidityScheduleEndTime =
dataset.getValiditySchedule().getEndTime();
+ if (StringUtils.isEmpty(datasetValidityScheduleEndTime)) {
+ return false;
+ }
+
+ try {
+ Date datasetExpiryTime =
DATE_FORMATTER.get().parse(datasetValidityScheduleEndTime);
+ return
validityScheduleEvaluator.isApplicable(datasetExpiryTime.getTime());
+ } catch (ParseException pe) {
+ failures.add(new ValidationFailureDetails(0, "endTime", "", false,
true, false, "Error parsing endTime:" + datasetValidityScheduleEndTime));
+ }
+ return false;
+ }
+
+ private RangerValiditySchedule
validateValidityCheckFilter(RangerValiditySchedule validityCheckFilter,
List<ValidationFailureDetails> failures) {
+ String startTime = validityCheckFilter.getStartTime();
+ String endTime = validityCheckFilter.getEndTime();
+ String timeZone = validityCheckFilter.getTimeZone();
+
+ if (StringUtils.isEmpty(startTime) && StringUtils.isEmpty(endTime)) {
+ return null;
+ }
+
+ if (StringUtils.isEmpty(startTime) || StringUtils.isEmpty(endTime)) {
+ failures.add(new ValidationFailureDetails(0, "startTime,endTime",
"", true, true, false, "empty values"));
+ return null;
+ }
+
+ if (StringUtils.isEmpty(timeZone)) {
+ validityCheckFilter.setTimeZone(SearchFilter.DEFAULT_TIME_ZONE);
+ }
+
+ RangerValidityScheduleValidator validator = new
RangerValidityScheduleValidator(validityCheckFilter);
+
+ return validator.validate(failures);
+ }
+
private PList<RangerDataset>
applyPaginataionAndSorting(List<RangerDataset> datasets, SearchFilter filter) {
int maxRows = filter.getMaxRows();
int startIndex = filter.getStartIndex();
diff --git
a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
index 6c93d71040..6833048e66 100755
---
a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
+++
b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
@@ -150,6 +150,9 @@ public SearchFilter getSearchFilter(@Nonnull
HttpServletRequest request, List<So
ret.setParam(SearchFilter.UPDATE_TIME_END,
request.getParameter(SearchFilter.UPDATE_TIME_END));
ret.setParam(SearchFilter.RESOURCE_CONTAINS,
request.getParameter(SearchFilter.RESOURCE_CONTAINS));
ret.setParam(SearchFilter.SHARED_WITH_ME,
request.getParameter(SearchFilter.SHARED_WITH_ME));
+ ret.setParam(SearchFilter.VALIDITY_EXPIRY_START,
request.getParameter(SearchFilter.VALIDITY_EXPIRY_START));
+ ret.setParam(SearchFilter.VALIDITY_EXPIRY_END,
request.getParameter(SearchFilter.VALIDITY_EXPIRY_END));
+ ret.setParam(SearchFilter.VALIDITY_TIME_ZONE,
request.getParameter(SearchFilter.VALIDITY_TIME_ZONE));
extractCommonCriteriasForFilter(request, ret, sortFields);
diff --git
a/security-admin/src/test/java/org/apache/ranger/rest/TestGdsREST.java
b/security-admin/src/test/java/org/apache/ranger/rest/TestGdsREST.java
index 11df009a46..143cfe9a90 100644
--- a/security-admin/src/test/java/org/apache/ranger/rest/TestGdsREST.java
+++ b/security-admin/src/test/java/org/apache/ranger/rest/TestGdsREST.java
@@ -16,11 +16,15 @@
*/
package org.apache.ranger.rest;
+import org.apache.ranger.biz.GdsDBStore;
+import org.apache.ranger.common.RESTErrorUtil;
import org.apache.ranger.common.RangerSearchUtil;
import org.apache.ranger.plugin.model.RangerGds;
import org.apache.ranger.plugin.model.RangerGrant;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerPrincipal;
+import org.apache.ranger.plugin.model.RangerValiditySchedule;
+import org.apache.ranger.plugin.util.SearchFilter;
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -31,7 +35,10 @@
import org.mockito.junit.MockitoJUnitRunner;
import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.WebApplicationException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -42,6 +49,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
@@ -52,7 +60,11 @@ public class TestGdsREST {
@Mock
RangerSearchUtil searchUtil;
@InjectMocks
- private GdsREST gdsREST = new GdsREST();
+ private GdsREST gdsREST;
+ @InjectMocks
+ GdsDBStore gdsDBStore;
+ @Mock
+ RESTErrorUtil restErrorUtil;
@Test
public void testAddDataSetGrants() {
@@ -207,6 +219,116 @@ public void
testGetDataSetGrantsByPrincipalAndAccessType() {
assertTrue("Grants for Principals: " +
Arrays.toString(requestedPrincipals) + " and AccessTypes: " +
Arrays.toString(nonexistentRequestedAccessTypes) + " should be empty",
updatedPolicyItemsByAccessType.isEmpty());
}
+ @Test
+ public void testSearchDataSetsByValidityPeriod() {
+ List<RangerGds.RangerDataset> rangerDatasets = new ArrayList<>();
+
+ RangerGds.RangerDataset rangerDataset1 = createRangerDataSet();
+ updateDatasetValiditySchedule(rangerDataset1, -5, -1);
+
+ RangerGds.RangerDataset rangerDataset2 = createRangerDataSet();
+ updateDatasetValiditySchedule(rangerDataset2, -5, 5);
+
+ RangerGds.RangerDataset rangerDataset3 = createRangerDataSet();
+ updateDatasetValiditySchedule(rangerDataset3, -10, 2);
+
+ RangerGds.RangerDataset rangerDataset4 = createRangerDataSet();
+ updateDatasetValiditySchedule(rangerDataset4, -2, 15);
+
+ RangerGds.RangerDataset rangerDataset5 = createRangerDataSet();
+ updateDatasetValiditySchedule(rangerDataset5, 5, 15);
+
+ RangerGds.RangerDataset rangerDataset6 = createRangerDataSet();
+ updateDatasetValiditySchedule(rangerDataset6, -15, -5);
+
+ rangerDatasets.addAll(Arrays.asList(rangerDataset1, rangerDataset2,
rangerDataset3, rangerDataset4, rangerDataset5, rangerDataset6));
+ List<RangerGds.RangerDataset> actualDatasets = new
ArrayList<>(rangerDatasets);
+
+ SearchFilter filter = new SearchFilter();
+ filter.setParam(SearchFilter.VALIDITY_TIME_ZONE,
SearchFilter.DEFAULT_TIME_ZONE);
+
+ //ValiditySchedule Filter criteria-1
+ String startTime = getFormattedDateString(-10);
+ String endTime = getFormattedDateString(-2);
+ filter.setParam(SearchFilter.VALIDITY_EXPIRY_START, startTime);
+ filter.setParam(SearchFilter.VALIDITY_EXPIRY_END, endTime);
+
+ List<RangerGds.RangerDataset> expectedDatasets =
Arrays.asList(rangerDataset6);
+
+ gdsDBStore.filterDatasetsByValidityExpiration(filter, actualDatasets);
+
+ assertEquals("Datasets expiry count mismatch between " + startTime + "
and " + endTime, expectedDatasets.size(), actualDatasets.size());
+
+ assertTrue("Mismatch in datasets returned for expiry between " +
startTime + " and " + endTime,
+ actualDatasets.containsAll(expectedDatasets));
+
+ //ValiditySchedule Filter criteria-2
+ actualDatasets.clear();
+ actualDatasets.addAll(rangerDatasets);
+
+ startTime = getFormattedDateString(-4);
+ endTime = getFormattedDateString(20);
+ filter.setParam(SearchFilter.VALIDITY_EXPIRY_START, startTime);
+ filter.setParam(SearchFilter.VALIDITY_EXPIRY_END, endTime);
+
+ expectedDatasets = Arrays.asList(rangerDataset1, rangerDataset2,
rangerDataset3, rangerDataset4, rangerDataset5);
+
+ gdsDBStore.filterDatasetsByValidityExpiration(filter, actualDatasets);
+
+ assertEquals("Datasets expiry count mismatch between " + startTime + "
and " + endTime, expectedDatasets.size(), actualDatasets.size());
+
+ assertTrue("Mismatch in datasets returned for expiry between " +
startTime + " and " + endTime,
+ actualDatasets.containsAll(expectedDatasets));
+
+ //ValiditySchedule Filter criteria-3
+ actualDatasets.clear();
+ actualDatasets.addAll(rangerDatasets);
+
+ startTime = getFormattedDateString(-15);
+ endTime = getFormattedDateString(0);
+ filter.setParam(SearchFilter.VALIDITY_EXPIRY_START, startTime);
+ filter.setParam(SearchFilter.VALIDITY_EXPIRY_END, endTime);
+
+ expectedDatasets = Arrays.asList(rangerDataset1, rangerDataset6);
+
+ gdsDBStore.filterDatasetsByValidityExpiration(filter, actualDatasets);
+
+ assertEquals("Datasets expiry count mismatch between " + startTime + "
and " + endTime, expectedDatasets.size(), actualDatasets.size());
+
+ assertTrue("Mismatch in datasets returned for expiry between " +
startTime + " and " + endTime,
+ actualDatasets.containsAll(expectedDatasets));
+
+ //ValiditySchedule Filter criteria-4 with invalid date
+ actualDatasets.clear();
+ actualDatasets.addAll(rangerDatasets);
+
+ startTime = getInvalidDateString(-5);
+ endTime = getFormattedDateString(16);
+ filter.setParam(SearchFilter.VALIDITY_EXPIRY_START, startTime);
+ filter.setParam(SearchFilter.VALIDITY_EXPIRY_END, endTime);
+
+
Mockito.when(restErrorUtil.createRESTException(Mockito.anyString())).thenThrow(new
WebApplicationException());
+
+ assertThrows(WebApplicationException.class, () -> {
+ gdsDBStore.filterDatasetsByValidityExpiration(filter,
actualDatasets);
+ });
+
+ //ValiditySchedule Filter criteria-5 without start and end time
+ actualDatasets.clear();
+ actualDatasets.addAll(rangerDatasets);
+
+ filter.removeParam(SearchFilter.VALIDITY_EXPIRY_START);
+ filter.removeParam(SearchFilter.VALIDITY_EXPIRY_END);
+
+ expectedDatasets = rangerDatasets;
+
+ gdsDBStore.filterDatasetsByValidityExpiration(filter, actualDatasets);
+
+ assertEquals("Datasets expiry count mismatch with empty start and end
time", expectedDatasets.size(), actualDatasets.size());
+
+ assertTrue("Mismatch in datasets returned with empty expiry time
range", actualDatasets.containsAll(expectedDatasets));
+ }
+
private RangerGds.RangerDataset createRangerDataSet() {
long id = new Random().nextInt(100);
RangerGds.RangerDataset dataset = new RangerGds.RangerDataset();
@@ -217,6 +339,31 @@ private RangerGds.RangerDataset createRangerDataSet() {
return dataset;
}
+ private RangerGds.RangerDataset
updateDatasetValiditySchedule(RangerGds.RangerDataset dataset, int
pastDaysToStart, int futureDaysToEnd) {
+ String start = getFormattedDateString(pastDaysToStart);
+ String end = getFormattedDateString(futureDaysToEnd);
+ String timezone = SearchFilter.DEFAULT_TIME_ZONE;
+ RangerValiditySchedule validitySchedule = new
RangerValiditySchedule(start, end, timezone, null);
+ dataset.setValiditySchedule(validitySchedule);
+
+ return dataset;
+ }
+
+ private String getFormattedDateString(int days) {
+ return getDateString(days,
RangerValiditySchedule.VALIDITY_SCHEDULE_DATE_STRING_SPECIFICATION);
+ }
+
+ private String getInvalidDateString(int days) {
+ return getDateString(days, "yyyy/MM/dd");
+ }
+
+ private String getDateString(int days, String pattern) {
+ DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern(pattern);
+ LocalDateTime current = LocalDateTime.now();
+
+ return current.plusDays(days).format(dateFormatter);
+ }
+
private RangerPolicy createPolicyForDataSet(RangerGds.RangerDataset
dataset) {
RangerPolicy policy = new RangerPolicy();
policy.setName("DATASET: " + dataset.getName() + "@" +
System.currentTimeMillis());
