This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch RANGER-3923 in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/RANGER-3923 by this push: new b57e62594 RANGER-4526: RangerBasePlugin updates to support GDS in disconnected mode b57e62594 is described below commit b57e6259401b178569dba56ef5a2a0587b14f14b Author: Madhan Neethiraj <mad...@apache.org> AuthorDate: Thu Nov 9 01:18:00 2023 -0800 RANGER-4526: RangerBasePlugin updates to support GDS in disconnected mode --- .../policyengine/RangerPolicyEngineImpl.java | 2 + .../policyengine/RangerPolicyEngineOptions.java | 10 + .../policyengine/RangerPolicyRepository.java | 12 + .../plugin/policyengine/gds/GdsPolicyEngine.java | 2 + .../ranger/plugin/service/RangerBasePlugin.java | 30 +- .../policyengine/gds/TestGdsPolicyEngine.java | 8 + .../plugin/service/TestRangerBasePlugin.java | 199 +++++++++++ .../src/test/resources/plugin/hive_gds_info.json | 141 ++++++++ .../src/test/resources/plugin/hive_policies.json | 161 +++++++++ .../src/test/resources/plugin/hive_roles.json | 15 + .../src/test/resources/plugin/hive_tags.json | 37 ++ .../src/test/resources/plugin/hive_user_store.json | 8 + .../resources/plugin/test_base_plugin_hive.json | 375 +++++++++++++++++++++ .../resources/policyengine/gds/gds_info_hive.json | 153 +++++++++ .../gds/test_gds_policy_engine_hive.json | 184 +--------- .../org/apache/ranger/sizing/RangerMemSizing.java | 85 ++++- 16 files changed, 1244 insertions(+), 178 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index 2d576c580..252482c8e 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@ -768,6 +768,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { ret.setIsAllowed(false); } else if (isAllowedByTags) { ret.setIsAllowed(true); + } else { + updateFromGdsResult(ret); } if (!ret.getIsAllowed() && !getIsFallbackSupported()) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java index a222eefef..f5f412797 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineOptions.java @@ -32,6 +32,7 @@ public class RangerPolicyEngineOptions { public boolean disablePolicyRefresher = false; public boolean disableTagRetriever = false; public boolean disableUserStoreRetriever = false; + public boolean disableGdsInfoRetriever = false; public boolean cacheAuditResults = true; public boolean evaluateDelegateAdminOnly = false; public boolean enableTagEnricherWithLocalRefresher = false; @@ -56,6 +57,7 @@ public class RangerPolicyEngineOptions { this.disablePolicyRefresher = other.disablePolicyRefresher; this.disableTagRetriever = other.disableTagRetriever; this.disableUserStoreRetriever = other.disableUserStoreRetriever; + this.disableGdsInfoRetriever = other.disableGdsInfoRetriever; this.cacheAuditResults = other.cacheAuditResults; this.evaluateDelegateAdminOnly = other.evaluateDelegateAdminOnly; this.enableTagEnricherWithLocalRefresher = other.enableTagEnricherWithLocalRefresher; @@ -83,6 +85,7 @@ public class RangerPolicyEngineOptions { disablePolicyRefresher = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.policy.refresher", false); disableTagRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tag.retriever", false); disableUserStoreRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.userstore.retriever", false); + disableGdsInfoRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.gdsinfo.retriever", false); cacheAuditResults = conf.getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", true); @@ -109,6 +112,7 @@ public class RangerPolicyEngineOptions { disablePolicyRefresher = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.policy.refresher", true); disableTagRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tag.retriever", true); disableUserStoreRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.userstore.retriever", true); + disableGdsInfoRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.gdsinfo.retriever", true); cacheAuditResults = false; evaluateDelegateAdminOnly = false; @@ -127,6 +131,7 @@ public class RangerPolicyEngineOptions { disablePolicyRefresher = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.policy.refresher", true); disableTagRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tag.retriever", true); disableUserStoreRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.userstore.retriever", true); + disableGdsInfoRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.gdsinfo.retriever", true); optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false); cacheAuditResults = false; @@ -143,6 +148,7 @@ public class RangerPolicyEngineOptions { disablePolicyRefresher = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.policy.refresher", true); disableTagRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.tag.retriever", false); disableUserStoreRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.userstore.retriever", false); + disableGdsInfoRetriever = conf.getBoolean(propertyPrefix + ".policyengine.option.disable.gdsinfo.retriever", false); optimizeTrieForRetrieval = conf.getBoolean(propertyPrefix + ".policyengine.option.optimize.trie.for.retrieval", false); cacheAuditResults = false; @@ -180,6 +186,7 @@ public class RangerPolicyEngineOptions { && this.disablePolicyRefresher == that.disablePolicyRefresher && this.disableTagRetriever == that.disableTagRetriever && this.disableUserStoreRetriever == that.disableUserStoreRetriever + && this.disableGdsInfoRetriever == that.disableGdsInfoRetriever && this.cacheAuditResults == that.cacheAuditResults && this.evaluateDelegateAdminOnly == that.evaluateDelegateAdminOnly && this.enableTagEnricherWithLocalRefresher == that.enableTagEnricherWithLocalRefresher @@ -211,6 +218,8 @@ public class RangerPolicyEngineOptions { ret *= 2; ret += disableUserStoreRetriever ? 1 : 0; ret *= 2; + ret += disableGdsInfoRetriever ? 1 : 0; + ret *= 2; ret += cacheAuditResults ? 1 : 0; ret *= 2; ret += evaluateDelegateAdminOnly ? 1 : 0; @@ -243,6 +252,7 @@ public class RangerPolicyEngineOptions { ", disablePolicyRefresher: " + disablePolicyRefresher + ", disableTagRetriever: " + disableTagRetriever + ", disableUserStoreRetriever: " + disableUserStoreRetriever + + ", disableGdsInfoRetriever: " + disableGdsInfoRetriever + ", enableTagEnricherWithLocalRefresher: " + enableTagEnricherWithLocalRefresher + ", enableUserStoreEnricherWithLocalRefresher: " + enableUserStoreEnricherWithLocalRefresher + ", disableTrieLookupPrefilter: " + disableTrieLookupPrefilter + diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index e02d3a32d..c3e7cc23a 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -25,6 +25,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.ranger.authorization.utils.JsonUtils; import org.apache.ranger.plugin.contextenricher.RangerAbstractContextEnricher; import org.apache.ranger.plugin.contextenricher.RangerContextEnricher; +import org.apache.ranger.plugin.contextenricher.RangerGdsEnricher; import org.apache.ranger.plugin.contextenricher.RangerTagEnricher; import org.apache.ranger.plugin.contextenricher.RangerTagForEval; import org.apache.ranger.plugin.contextenricher.RangerUserStoreEnricher; @@ -59,6 +60,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import static org.apache.ranger.plugin.contextenricher.RangerGdsEnricher.RETRIEVER_CLASSNAME_OPTION; import static org.apache.ranger.plugin.contextenricher.RangerTagEnricher.TAG_RETRIEVER_CLASSNAME_OPTION; import static org.apache.ranger.plugin.contextenricher.RangerUserStoreEnricher.USERSTORE_RETRIEVER_CLASSNAME_OPTION; import static org.apache.ranger.plugin.policyengine.RangerPolicyEngine.PLUGIN_AUDIT_FILTER; @@ -1009,6 +1011,16 @@ public class RangerPolicyRepository { } } + if (options.disableGdsInfoRetriever && StringUtils.equals(enricherDef.getEnricher(), RangerGdsEnricher.class.getName())) { + if (MapUtils.isNotEmpty(enricherDef.getEnricherOptions())) { + Map<String, String> enricherOptions = new HashMap<>(enricherDef.getEnricherOptions()); + + enricherOptions.remove(RETRIEVER_CLASSNAME_OPTION); + + enricherDef = new RangerServiceDef.RangerContextEnricherDef(enricherDef.getItemId(), enricherDef.getName(), enricherDef.getEnricher(), enricherOptions); + } + } + if (!options.disableContextEnrichers) { RangerServiceDef.RangerContextEnricherDef contextEnricherDef = enricherDef; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsPolicyEngine.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsPolicyEngine.java index d9aa81732..5f9f1f90e 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsPolicyEngine.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsPolicyEngine.java @@ -114,6 +114,8 @@ public class GdsPolicyEngine { } } + ret.finalizeAcls(); + return ret; } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index 748efd681..db377c970 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -108,10 +108,14 @@ public class RangerBasePlugin { } public RangerBasePlugin(RangerPluginConfig pluginConfig, ServicePolicies policies, ServiceTags tags, RangerRoles roles) { - this(pluginConfig, policies, tags, roles, null); + this(pluginConfig, policies, tags, roles, null, null); } public RangerBasePlugin(RangerPluginConfig pluginConfig, ServicePolicies policies, ServiceTags tags, RangerRoles roles, RangerUserStore userStore) { + this(pluginConfig, policies, tags, roles, userStore, null); + } + + public RangerBasePlugin(RangerPluginConfig pluginConfig, ServicePolicies policies, ServiceTags tags, RangerRoles roles, RangerUserStore userStore, ServiceGdsInfo gdsInfo) { this(pluginConfig); init(); @@ -138,6 +142,16 @@ public class RangerBasePlugin { LOG.warn("RangerBasePlugin(userStoreVersion=" + userStore.getUserStoreVersion() + "): no userstore enricher found. Plugin will not enforce user/group attribute-based policies"); } } + + if (gdsInfo != null) { + RangerGdsEnricher gdsEnricher = getGdsEnricher(); + + if (gdsEnricher != null) { + gdsEnricher.setGdsInfo(gdsInfo); + } else { + LOG.warn("RangerBasePlugin(gdsInfo=" + gdsInfo.getGdsVersion() + "): no GDS enricher found. Plugin will not enforce GDS policies"); + } + } } public static AuditHandler getAuditProvider(String serviceName) { @@ -1154,8 +1168,8 @@ public class RangerBasePlugin { return ret; } - public GdsPolicyEngine getGdsPolicyEngine() { - GdsPolicyEngine ret = null; + public RangerGdsEnricher getGdsEnricher() { + RangerGdsEnricher ret = null; RangerAuthContext authContext = getCurrentRangerAuthContext(); if (authContext != null) { @@ -1166,9 +1180,7 @@ public class RangerBasePlugin { for (RangerContextEnricher enricher : contextEnrichers) { if (enricher instanceof RangerGdsEnricher) { - RangerGdsEnricher gdsEnricher = (RangerGdsEnricher) enricher; - - ret = gdsEnricher.getGdsPolicyEngine(); + ret = (RangerGdsEnricher) enricher; break; } @@ -1179,6 +1191,12 @@ public class RangerBasePlugin { return ret; } + public GdsPolicyEngine getGdsPolicyEngine() { + RangerGdsEnricher gdsEnricher = getGdsEnricher(); + + return gdsEnricher != null ? gdsEnricher.getGdsPolicyEngine() : null; + } + public static RangerResourceACLs getMergedResourceACLs(RangerResourceACLs baseACLs, RangerResourceACLs chainedACLs) { if (LOG.isDebugEnabled()) { LOG.debug("==> RangerBasePlugin.getMergedResourceACLs()"); diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/gds/TestGdsPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/gds/TestGdsPolicyEngine.java index b5ca483ad..9d1560562 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/gds/TestGdsPolicyEngine.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/gds/TestGdsPolicyEngine.java @@ -20,6 +20,7 @@ package org.apache.ranger.plugin.policyengine.gds; import com.google.gson.*; +import org.apache.commons.lang.StringUtils; import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.validation.RangerServiceDefHelper; @@ -67,6 +68,12 @@ public class TestGdsPolicyEngine { private void runTests(Reader reader, String testName) { GdsPolicyEngineTestCase testCase = gsonBuilder.fromJson(reader, GdsPolicyEngineTestCase.class); + if (StringUtils.isNotBlank(testCase.gdsInfoFilename)) { + InputStream inStream = this.getClass().getResourceAsStream(testCase.gdsInfoFilename); + + testCase.gdsInfo = gsonBuilder.fromJson(new InputStreamReader(inStream), ServiceGdsInfo.class); + } + assertTrue("invalid input: " + testName, testCase != null && testCase.gdsInfo != null && testCase.tests != null); testCase.serviceDef.setMarkerAccessTypes(ServiceDefUtil.getMarkerAccessTypes(testCase.serviceDef.getAccessTypes())); @@ -98,6 +105,7 @@ public class TestGdsPolicyEngine { public RangerServiceDef serviceDef; public Map<String, SecurityZoneInfo> securityZones; public ServiceGdsInfo gdsInfo; + public String gdsInfoFilename; public List<TestData> tests; } diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/service/TestRangerBasePlugin.java b/agents-common/src/test/java/org/apache/ranger/plugin/service/TestRangerBasePlugin.java new file mode 100644 index 000000000..9886f25c0 --- /dev/null +++ b/agents-common/src/test/java/org/apache/ranger/plugin/service/TestRangerBasePlugin.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.service; + +import com.google.gson.*; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig; +import org.apache.ranger.plugin.policyengine.*; +import org.apache.ranger.plugin.util.*; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.lang.reflect.Type; +import java.util.*; + +import static org.junit.Assert.*; + +public class TestRangerBasePlugin { + static Gson gsonBuilder; + static RangerPolicyEngineOptions peOptions; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSSZ") + .setPrettyPrinting() + .registerTypeAdapter(RangerAccessRequest.class, new RangerAccessRequestDeserializer()) + .registerTypeAdapter(RangerAccessResource.class, new RangerResourceDeserializer()) + .create(); + + peOptions = new RangerPolicyEngineOptions(); + + peOptions.disablePolicyRefresher = true; + peOptions.disableTagRetriever = true; + peOptions.disableUserStoreRetriever = true; + peOptions.disableGdsInfoRetriever = true; + + } + + + @Test + public void testBasePluginHive() { + runTestsFromResourceFile("/plugin/test_base_plugin_hive.json"); + } + + private void runTestsFromResourceFile(String resourceFile) { + InputStream inStream = this.getClass().getResourceAsStream(resourceFile); + InputStreamReader reader = new InputStreamReader(inStream); + + runTests(reader, resourceFile); + } + + private void runTests(Reader reader, String testName) { + RangerBasePluginTestCase testCase = readTestCase(reader); + + assertNotNull("invalid input: " + testName, testCase); + assertNotNull("invalid input: " + testName, testCase.policies); + assertNotNull("invalid input: " + testName, testCase.tags); + assertNotNull("invalid input: " + testName, testCase.roles); + assertNotNull("invalid input: " + testName, testCase.userStore); + assertNotNull("invalid input: " + testName, testCase.gdsInfo); + assertNotNull("invalid input: " + testName, testCase.tests); + + RangerPluginConfig pluginConfig = new RangerPluginConfig(testCase.policies.getServiceDef().getName(), testCase.policies.getServiceName(), "hive", "cl1", "on-prem", peOptions); + RangerBasePlugin plugin = new RangerBasePlugin(pluginConfig, testCase.policies, testCase.tags, testCase.roles, testCase.userStore, testCase.gdsInfo); + + for (TestData test : testCase.tests) { + RangerAccessRequest request = test.request; + + if (test.result != null) { + RangerAccessResult result = plugin.isAccessAllowed(request); + + assertNotNull("result was null! - " + test.name, result); + assertEquals("isAllowed mismatched! - " + test.name, test.result.getIsAllowed(), result.getIsAllowed()); + assertEquals("isAccessDetermined mismatched! - " + test.name, test.result.getIsAccessDetermined(), result.getIsAccessDetermined()); + assertEquals("isAllowed mismatched! - " + test.name, test.result.getPolicyId(), result.getPolicyId()); + assertEquals("isAudited mismatched! - " + test.name, test.result.getIsAudited(), result.getIsAudited()); + assertEquals("isAuditedDetermined mismatched! - " + test.name, test.result.getIsAuditedDetermined(), result.getIsAuditedDetermined()); + } + + if (test.acls != null) { + RangerAccessRequest req = new RangerAccessRequestImpl(request.getResource(), RangerPolicyEngine.ANY_ACCESS, null, null, null); + RangerResourceACLs acls = plugin.getResourceACLs(req); + + assertEquals(test.name, test.acls, acls); + } + } + } + + private RangerBasePluginTestCase readTestCase(Reader reader) { + RangerBasePluginTestCase testCase = gsonBuilder.fromJson(reader, RangerBasePluginTestCase.class); + + if (StringUtils.isNotBlank(testCase.policiesFilename)) { + InputStream inStream = this.getClass().getResourceAsStream(testCase.policiesFilename); + + testCase.policies = gsonBuilder.fromJson(new InputStreamReader(inStream), ServicePolicies.class); + } + + if (StringUtils.isNotBlank(testCase.tagsFilename)) { + InputStream inStream = this.getClass().getResourceAsStream(testCase.tagsFilename); + + testCase.tags = gsonBuilder.fromJson(new InputStreamReader(inStream), ServiceTags.class); + } + + if (StringUtils.isNotBlank(testCase.rolesFilename)) { + InputStream inStream = this.getClass().getResourceAsStream(testCase.rolesFilename); + + testCase.roles = gsonBuilder.fromJson(new InputStreamReader(inStream), RangerRoles.class); + } + + if (StringUtils.isNotBlank(testCase.userStoreFilename)) { + InputStream inStream = this.getClass().getResourceAsStream(testCase.userStoreFilename); + + testCase.userStore = gsonBuilder.fromJson(new InputStreamReader(inStream), RangerUserStore.class); + } + + if (StringUtils.isNotBlank(testCase.gdsInfoFilename)) { + InputStream inStream = this.getClass().getResourceAsStream(testCase.gdsInfoFilename); + + testCase.gdsInfo = gsonBuilder.fromJson(new InputStreamReader(inStream), ServiceGdsInfo.class); + } + + if (testCase.policies != null && testCase.policies.getServiceDef() != null) { + testCase.policies.getServiceDef().setMarkerAccessTypes(ServiceDefUtil.getMarkerAccessTypes(testCase.policies.getServiceDef().getAccessTypes())); + } + + return testCase; + } + + static class RangerBasePluginTestCase { + public ServicePolicies policies; + public ServiceTags tags; + public RangerRoles roles; + public RangerUserStore userStore; + public ServiceGdsInfo gdsInfo; + public String policiesFilename; + public String tagsFilename; + public String rolesFilename; + public String userStoreFilename; + public String gdsInfoFilename; + public List<TestData> tests; + } + + static class TestData { + public String name; + public RangerAccessRequest request; + public RangerAccessResult result; + public RangerResourceACLs acls; + } + + static class RangerAccessRequestDeserializer implements JsonDeserializer<RangerAccessRequest> { + @Override + public RangerAccessRequest deserialize(JsonElement jsonObj, Type type, + JsonDeserializationContext context) throws JsonParseException { + RangerAccessRequestImpl ret = gsonBuilder.fromJson(jsonObj, RangerAccessRequestImpl.class); + + ret.setAccessType(ret.getAccessType()); // to force computation of isAccessTypeAny and isAccessTypeDelegatedAdmin + if (ret.getAccessTime() == null) { + ret.setAccessTime(new Date()); + } + Map<String, Object> reqContext = ret.getContext(); + Object accessTypes = reqContext.get("ACCESSTYPES"); + if (accessTypes != null) { + Collection<String> accessTypesCollection = (Collection<String>) accessTypes; + Set<String> requestedAccesses = new HashSet<>(accessTypesCollection); + ret.getContext().put("ACCESSTYPES", requestedAccesses); + } + + return ret; + } + } + + static class RangerResourceDeserializer implements JsonDeserializer<RangerAccessResource> { + @Override + public RangerAccessResource deserialize(JsonElement jsonObj, Type type, + JsonDeserializationContext context) throws JsonParseException { + return gsonBuilder.fromJson(jsonObj, RangerAccessResourceImpl.class); + } + } +} diff --git a/agents-common/src/test/resources/plugin/hive_gds_info.json b/agents-common/src/test/resources/plugin/hive_gds_info.json new file mode 100644 index 000000000..f7797cf92 --- /dev/null +++ b/agents-common/src/test/resources/plugin/hive_gds_info.json @@ -0,0 +1,141 @@ +{ + "serviceName": "dev_hive", + "datasets": [ + { "id": 1, "name": "dataset-1", + "policies": [ + { "id": 2001, "name": "dataset-1", "isEnabled": true, "isAuditEnabled": true, + "resources": { "dataset-id": { "values": ["1"] } }, + "policyItems":[ + { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "ds-user", "ds1-user" ], "groups": []} + ] + } + ] + }, + { "id": 2, "name": "dataset-2", + "policies": [ + { "id": 2002, "name": "dataset-2", "isEnabled": true, "isAuditEnabled": true, + "resources": { "dataset-id": { "values": ["2"] } }, + "policyItems":[ + { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "ds-user", "ds2-user" ], "groups": []} + ] + } + ] + }, + { "id": 3, "name": "dataset-3", + "policies": [ + { "id": 2003, "name": "dataset-3", "isEnabled": true, "isAuditEnabled": true, + "resources": { "dataset-id": { "values": ["3"] } }, + "policyItems":[ + { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "ds-user", "ds3-user" ], "groups": []} + ] + } + ] + }, + { "id": 4, "name": "dataset-4", + "policies": [ + { "id": 2004, "name": "dataset-4", "isEnabled": true, "isAuditEnabled": true, + "resources": { "dataset-id": { "values": ["4"] } }, + "policyItems":[ + { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "ds-user", "ds4-user" ], "groups": []} + ] + } + ] + } + ], + "projects": [ + { "id": 1, "name": "project-1", + "policies": [ + { "id": 3001, "name": "project-1", "isEnabled": true, "isAuditEnabled": true, + "resources": { "project-id": { "values": ["1"] } }, + "policyItems":[ + { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "proj-user", "proj1-user" ], "groups": []} + ] + } + ] + }, + { "id": 2, "name": "project-2", + "policies": [ + { "id": 3002, "name": "project-2", "isEnabled": true, "isAuditEnabled": true, + "resources": { "project-id": { "values": ["2"] } }, + "policyItems":[ + { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "proj-user", "proj2-user" ], "groups": []} + ] + } + ] + } + ], + "dataShares": [ + { "id": 1, "name": "hive-sales-2023", "zoneName": "sales", "conditionExpr": "", "defaultAccessTypes": [ "_READ" ], "defaultTagMasks": [ ] }, + { "id": 2, "name": "hive-finance-2023", "zoneName": "finance", "conditionExpr": "", "defaultAccessTypes": [ "_READ" ], "defaultTagMasks": [ ] }, + { "id": 3, "name": "hive-shipping-2023", "zoneName": "shipping", "conditionExpr": "", "defaultAccessTypes": [ "_READ" ], "defaultTagMasks": [ ] }, + { "id": 4, "name": "hive-new-customers-2023", "zoneName": "", "conditionExpr": "", "defaultAccessTypes": [ "_READ" ], "defaultTagMasks": [ ] }, + { "id": 5, "name": "hive-facilities", "zoneName": "", "conditionExpr": "", "defaultAccessTypes": [ "_READ" ], "defaultTagMasks": [ ] } + ], + "dshids": [ + { "dataShareId": 1, "datasetId": 1, "status": "ACTIVE", "validitySchedule": null }, + { "dataShareId": 2, "datasetId": 1, "status": "ACTIVE", "validitySchedule": null }, + { "dataShareId": 2, "datasetId": 2, "status": "ACTIVE", "validitySchedule": null }, + { "dataShareId": 3, "datasetId": 2, "status": "ACTIVE", "validitySchedule": null }, + { "dataShareId": 4, "datasetId": 3, "status": "ACTIVE", "validitySchedule": null }, + { "dataShareId": 5, "datasetId": 4, "status": "ACTIVE", "validitySchedule": null } + ], + "dips": [ + { "datasetId": 1, "projectId": 1, "status": "ACTIVE", "validitySchedule": null }, + { "datasetId": 2, "projectId": 1, "status": "ACTIVE", "validitySchedule": null }, + { "datasetId": 3, "projectId": 2, "status": "ACTIVE", "validitySchedule": null } + ], + "resources": [ + { + "id": 11, "dataShareId": 1, "conditionExpr": "", "accessTypes": [ "select" ], + "resource": { "database": { "values": [ "sales" ] }, "table": { "values": [ "prospects" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, + "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": { } + }, + { + "id": 12, "dataShareId": 1, "conditionExpr": "", "accessTypes": [ "select" ], + "resource": { "database": { "values": [ "sales" ] }, "table": { "values": [ "orders" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, + "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": { } + }, + { + "id": 21, "dataShareId": 2, "conditionExpr": "", "accessTypes": [ "select" ], + "resource": { "database": { "values": [ "finance" ] }, "table": { "values": [ "invoices" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, + "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": { } + }, + { + "id": 22, "dataShareId": 2, "conditionExpr": "", "accessTypes": [ "select" ], + "resource": { "database": { "values": [ "finance" ] }, "table": { "values": [ "payments" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, + "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": { } + }, + { + "id": 31, "dataShareId": 3, "conditionExpr": "", "accessTypes": [ "select" ], + "resource": { "database": { "values": [ "shipping" ] }, "table": { "values": [ "shipments" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, + "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": { } + }, + { + "id": 41, "dataShareId": 4, "conditionExpr": "", "accessTypes": [ "select" ], + "resource": { "database": { "values": [ "customers" ] }, "table": { "values": [ "contact_info" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, + "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": null + }, + { + "id": 51, "dataShareId": 5, "conditionExpr": "", "accessTypes": [ "select" ], + "resource": { "database": { "values": [ "operations" ] }, "table": { "values": [ "facilities" ] } }, "rowFilter": null, + "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": null + } + ], + "gdsServiceDef": { + "name": "gds", + "id": 3, + "resources": [ + { "name": "dataset-id", "level":1, "parent": "", "mandatory": true, "lookupSupported": false, "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", "matcherOptions":{ "wildCard": false, "ignoreCase": true }, "label": "Dataset ID", "description": "Dataset ID" }, + { "name": "project-id", "level":1, "parent": "", "mandatory": true, "lookupSupported": false, "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", "matcherOptions":{ "wildCard": false, "ignoreCase": true }, "label": "Project ID", "description": "Project ID" } + ], + "accessTypes":[ + { "itemId": 1, "name": "_CREATE", "label": "_CREATE" }, + { "itemId": 2, "name": "_READ", "label": "_READ" }, + { "itemId": 3, "name": "_UPDATE", "label": "_UPDATE" }, + { "itemId": 4, "name": "_DELETE", "label": "_DELETE" }, + { "itemId": 5, "name": "_MANAGE", "label": "_MANAGE" }, + { "itemId": 6, "name": "_ALL", "label": "_ALL" } + ] + }, + "gdsVersion": 1 +} \ No newline at end of file diff --git a/agents-common/src/test/resources/plugin/hive_policies.json b/agents-common/src/test/resources/plugin/hive_policies.json new file mode 100644 index 000000000..7eef385f5 --- /dev/null +++ b/agents-common/src/test/resources/plugin/hive_policies.json @@ -0,0 +1,161 @@ +{ + "serviceDef":{ + "name": "hive", + "id": 3, + "resources": [ + { "name": "database", "level":1, "parent": "", "mandatory": true, "lookupSupported": true, "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", "matcherOptions":{ "wildCard": true, "ignoreCase": true }, "label": "Hive Database", "description": "Hive Database" }, + { "name": "url", "level":1, "parent": "", "mandatory": true, "lookupSupported": false, "matcher": "org.apache.ranger.plugin.resourcematcher.RangerURLResourceMatcher", "matcherOptions":{ "wildCard": true, "ignoreCase": true }, "label": "URL", "description": "URL", "recursiveSupported": true }, + { "name": "hiveservice", "level":1, "parent": "", "mandatory": true, "lookupSupported": false, "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", "matcherOptions":{ "wildCard": true, "ignoreCase": true }, "label": "HiveService", "description": "HiveService" }, + { "name": "table", "level":2, "parent": "database", "mandatory": true, "lookupSupported": true, "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", "matcherOptions":{ "wildCard": true, "ignoreCase": true }, "label": "Hive Table", "description": "Hive Table" }, + { "name": "udf", "level":2, "parent": "database", "mandatory": true, "lookupSupported": true, "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", "matcherOptions":{ "wildCard": true, "ignoreCase": true }, "label": "Hive UDF", "description": "Hive UDF" }, + { "name": "column", "level":3, "parent": "table", "mandatory": true, "lookupSupported": true, "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", "matcherOptions":{ "wildCard": true, "ignoreCase": true }, "label": "Hive Column", "description": "Hive Column" } + ], + "accessTypes":[ + { "name": "select", "label": "Select", "category": "READ" }, + { "name": "update", "label": "Update", "category": "UPDATE" }, + { "name": "create", "label": "Create", "category": "CREATE" }, + { "name": "drop", "label": "Drop", "category": "DELETE" }, + { "name": "alter", "label": "Alter", "category": "CREATE" }, + { "name": "index", "label": "Index", "category": "MANAGE" }, + { "name": "lock", "label": "Lock", "category": "MANAGE" }, + { "name": "read", "label": "Read", "category": "READ" }, + { "name": "write", "label": "Write", "category": "UPDATE" }, + { "name": "repladmin", "label": "ReplAdmin", "category": "MANAGE" }, + { "name": "serviceadmin", "label": "ServiceAdmin", "category": "MANAGE" }, + { "name": "all", "label": "All", + "impliedGrants": [ "select", "update", "create", "drop", "alter", "index", "lock", "read", "write", "repladmin", "serviceadmin" ] + } + ] + }, + "securityZones": { + "sales": { + "zoneName": "sales", + "resources": [ + { "database": [ "sales" ] } + ], + "policies": [ + { "id": 100, "name": "table: sales.prospects", "isEnabled": true, "isAuditEnabled": true, + "resources": { "database": { "values": [ "sales" ] }, "table": { "values": [ "prospects" ] } }, + "policyItems":[ + { "accesses": [ { "type": "select", "isAllowed": true } ], "users": [ "res-user" ], "delegateAdmin": false } + ] + }, + { "id": 101, "name": "table: sales.orders", "isEnabled": true, "isAuditEnabled": true, + "resources": { "database": { "values": [ "sales" ] }, "table": { "values": [ "orders" ] } }, + "policyItems":[ + { "accesses": [ { "type": "select", "isAllowed": true } ], "users": [ "res-user" ], "delegateAdmin": false } + ] + } + ] + }, + "finance": { + "zoneName": "finance", + "resources": [ + { "database": [ "finance" ] } + ], + "policies": [ + { "id": 110, "name": "table: finance.invoices", "isEnabled": true, "isAuditEnabled": true, + "resources": { "database": { "values": [ "finance" ] }, "table": { "values": [ "invoices" ] } }, + "policyItems":[ + { "accesses": [ { "type": "select", "isAllowed": true } ], "users": [ "res-user" ], "delegateAdmin": false } + ] + }, + { "id": 111, "name": "table: finance.payments", "isEnabled": true, "isAuditEnabled": true, + "resources": { "database": { "values": [ "finance" ] }, "table": { "values": [ "payments" ] } }, + "policyItems":[ + { "accesses": [ { "type": "select", "isAllowed": true } ], "users": [ "res-user" ], "delegateAdmin": false } + ] + } + ] + }, + "shipping": { + "zoneName": "shipping", + "resources": [ + { "database": [ "shipping" ] } + ], + "policies": [ + { "id": 121, "name": "table: shipping.shipments", "isEnabled": true, "isAuditEnabled": true, + "resources": { "database": { "values": [ "shipping" ] }, "table": { "values": [ "shipments" ] } }, + "policyItems":[ + { "accesses": [ { "type": "select", "isAllowed": true } ], "users": [ "res-user" ], "delegateAdmin": false } + ] + } + ] + } + }, + "policies":[ + { "id": 131, "name": "table: customers.contact_info", "isEnabled": true, "isAuditEnabled": true, + "resources": { "database": { "values": [ "customers" ] }, "table": { "values": [ "contact_info" ] } }, + "policyItems":[ + { "accesses": [ { "type": "select", "isAllowed": true } ], "users": [ "res-user" ], "delegateAdmin": false } + ] + }, + { "id": 141, "name": "table: operations.facilities", "isEnabled": true, "isAuditEnabled": true, + "resources": { "database": { "values": [ "operations" ] }, "table": { "values": [ "facilities" ] } }, + "policyItems":[ + { "accesses": [ { "type": "select", "isAllowed": true } ], "users": [ "res-user" ], "delegateAdmin": false } + ] + } + ], + "tagPolicies": { + "serviceName": "dev_tag", + "serviceDef": { + "name": "tag", + "id": 1, + "resources": [ + { "name": "tag", "level":1, "parent": "", "mandatory": true, "lookupSupported": true, "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", "matcherOptions":{ "wildCard": true, "ignoreCase": true }, "label": "TAGe", "description": "TAG" } + ], + "accessTypes":[ + { "name": "hive:select", "label": "Select", "category": "READ" }, + { "name": "hive:update", "label": "Update", "category": "UPDATE" }, + { "name": "hive:create", "label": "Create", "category": "CREATE" }, + { "name": "hive:drop", "label": "Drop", "category": "DELETE" }, + { "name": "hive:alter", "label": "Alter", "category": "CREATE" }, + { "name": "hive:index", "label": "Index", "category": "MANAGE" }, + { "name": "hive:lock", "label": "Lock", "category": "MANAGE" }, + { "name": "hive:read", "label": "Read", "category": "READ" }, + { "name": "hive:write", "label": "Write", "category": "UPDATE" }, + { "name": "hive:repladmin", "label": "ReplAdmin", "category": "MANAGE" }, + { "name": "hive:serviceadmin", "label": "ServiceAdmin", "category": "MANAGE" }, + { "name": "hive:all", "label": "All", + "impliedGrants": [ "hive:select", "hive:update", "hive:create", "hive:drop", "hive:alter", "hive:index", "hive:lock", "hive:read", "hive:write", "hive:repladmin", "hive:serviceadmin" ] + } + ], + "contextEnrichers": [ + { "itemId": 1, "name": "TagEnricher", "enricher": "org.apache.ranger.plugin.contextenricher.RangerTagEnricher", "enricherOptions" : { "tagRetrieverClassName": "org.apache.ranger.plugin.contextenricher.RangerAdminTagRetriever", "tagRefresherPollingInterval": 60000 }} + ] + }, + "policies": [ + { "id": 200, "name": "tag: SALES", "isEnabled": true, "isAuditEnabled": true, + "resources": { "tag": { "values": [ "SALES" ] } }, + "policyItems":[ + { "accesses": [ { "type": "select", "isAllowed": true } ], "users": [ "tag-user" ], "delegateAdmin": false } + ] + }, + { "id": 201, "name": "tag: FINANCE", "isEnabled": true, "isAuditEnabled": true, + "resources": { "tag": { "values": [ "FINANCE" ] } }, + "policyItems":[ + { "accesses": [ { "type": "select", "isAllowed": true } ], "users": [ "tag-user" ], "delegateAdmin": false } + ] + }, + { "id": 202, "name": "tag: SHIPPING", "isEnabled": true, "isAuditEnabled": true, + "resources": { "tag": { "values": [ "SHIPPING" ] } }, + "policyItems":[ + { "accesses": [ { "type": "select", "isAllowed": true } ], "users": [ "tag-user" ], "delegateAdmin": false } + ] + }, + { "id": 203, "name": "tag: CUSTOMER", "isEnabled": true, "isAuditEnabled": true, + "resources": { "tag": { "values": [ "CUSTOMER" ] } }, + "policyItems":[ + { "accesses": [ { "type": "select", "isAllowed": true } ], "users": [ "tag-user" ], "delegateAdmin": false } + ] + }, + { "id": 204, "name": "tag: OPERATIONS", "isEnabled": true, "isAuditEnabled": true, + "resources": { "tag": { "values": [ "OPERATIONS" ] } }, + "policyItems":[ + { "accesses": [ { "type": "select", "isAllowed": true } ], "users": [ "tag-user" ], "delegateAdmin": false } + ] + } + ] + } +} diff --git a/agents-common/src/test/resources/plugin/hive_roles.json b/agents-common/src/test/resources/plugin/hive_roles.json new file mode 100644 index 000000000..c7c83e8ca --- /dev/null +++ b/agents-common/src/test/resources/plugin/hive_roles.json @@ -0,0 +1,15 @@ +{ + "serviceName": "dev_hive", + "rangerRoles": [ + { + "name": "role-1", + "users": [ + { "name": "user-1", "isAdmin": false }, + { "name": "user-2", "usAdmin": false } + ], + "groups": [ ], + "roles": [ ] + } + ], + "roleVersion": 1 +} \ No newline at end of file diff --git a/agents-common/src/test/resources/plugin/hive_tags.json b/agents-common/src/test/resources/plugin/hive_tags.json new file mode 100644 index 000000000..38eec4a31 --- /dev/null +++ b/agents-common/src/test/resources/plugin/hive_tags.json @@ -0,0 +1,37 @@ +{ + "op": "add_or_update", + "tagModel": "resource_private", + "serviceName": "dev_hive", + "tagDefinitions": { + "1": { "name": "SALES" }, + "2": { "name": "FINANCE" }, + "3": { "name": "SHIPPING" }, + "4": { "name": "CUSTOMER" }, + "5": { "name": "OPERATIONS"} + }, + "tags": { + "1": { "id": 1, "type": "SALES" }, + "2": { "id": 2, "type": "FINANCE" }, + "3": { "id": 3, "type": "SHIPPING" }, + "4": { "id": 4, "type": "CUSTOMER" }, + "5": { "id": 5, "type": "OPERATIONS" } + }, + "serviceResources": [ + { "id": 1, "serviceName": "dev_hive", "resourceElements": { "database": { "values": [ "sales" ] }, "table": { "values": [ "prospects" ] } } }, + { "id": 2, "serviceName": "dev_hive", "resourceElements": { "database": { "values": [ "sales" ] }, "table": { "values": [ "orders" ] } } }, + { "id": 3, "serviceName": "dev_hive", "resourceElements": { "database": { "values": [ "finance" ] }, "table": { "values": [ "invoices" ] } } }, + { "id": 4, "serviceName": "dev_hive", "resourceElements": { "database": { "values": [ "finance" ] }, "table": { "values": [ "payments" ] } } }, + { "id": 5, "serviceName": "dev_hive", "resourceElements": { "database": { "values": [ "shipping" ] }, "table": { "values": [ "shipments" ] } } }, + { "id": 6, "serviceName": "dev_hive", "resourceElements": { "database": { "values": [ "customers" ] }, "table": { "values": [ "contact_info" ] } } }, + { "id": 7, "serviceName": "dev_hive", "resourceElements": { "database": { "values": [ "operations" ] }, "table": { "values": [ "facilities" ] } } } + ], + "resourceToTagIds": { + "1": [ 1 ], + "2": [ 1 ], + "3": [ 2 ], + "4": [ 2 ], + "5": [ 3 ], + "6": [ 4 ], + "7": [ 5 ] + } +} diff --git a/agents-common/src/test/resources/plugin/hive_user_store.json b/agents-common/src/test/resources/plugin/hive_user_store.json new file mode 100644 index 000000000..8b4bfe88f --- /dev/null +++ b/agents-common/src/test/resources/plugin/hive_user_store.json @@ -0,0 +1,8 @@ +{ + "userAttrMapping": { }, + "groupAttrMapping": { }, + "userGroupMapping": { }, + "userCloudIdMapping": { }, + "groupCloudIdMapping": { }, + "userStoreVersion": 1 +} diff --git a/agents-common/src/test/resources/plugin/test_base_plugin_hive.json b/agents-common/src/test/resources/plugin/test_base_plugin_hive.json new file mode 100644 index 000000000..1e29f39c9 --- /dev/null +++ b/agents-common/src/test/resources/plugin/test_base_plugin_hive.json @@ -0,0 +1,375 @@ +{ + "policiesFilename": "/plugin/hive_policies.json", + "tagsFilename": "/plugin/hive_tags.json", + "rolesFilename": "/plugin/hive_roles.json", + "userStoreFilename": "/plugin/hive_user_store.json", + "gdsInfoFilename": "/plugin/hive_gds_info.json", + "tests": [ + { + "name": "table: sales.prospects, user: res-user, access: select", + "request": { + "resource": { "elements": { "database": "sales", "table": "prospects" } }, + "accessType": "select", "user": "res-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 100 } + }, + { + "name": "table: sales.prospects, user: tag-user, access: select", + "request": { + "resource": { "elements": { "database": "sales", "table": "prospects" } }, + "accessType": "select", "user": "tag-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 200 } + }, + { + "name": "table: sales.prospects, user: ds-user, access: select", + "request": { + "resource": { "elements": { "database": "sales", "table": "prospects" } }, + "accessType": "select", "user": "ds-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 2001 } + }, + { + "name": "table: sales.prospects, user: ds1-user, access: select", + "request": { + "resource": { "elements": { "database": "sales", "table": "prospects" } }, + "accessType": "select", "user": "ds1-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 2001 } + }, + { + "name": "table: sales.prospects, user: proj-user, access: select", + "request": { + "resource": { "elements": { "database": "sales", "table": "prospects" } }, + "accessType": "select", "user": "proj-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 3001 } + }, + { + "name": "table: sales.prospects, user: proj1-user, access: select", + "request": { + "resource": { "elements": { "database": "sales", "table": "prospects" } }, + "accessType": "select", "user": "proj1-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 3001 } + }, + { + "name": "table: sales.prospects, user: no-user, access: select", + "request": { + "resource": { "elements": { "database": "sales", "table": "prospects" } }, + "accessType": "select", "user": "no-user", "userGroups": [] + }, + "result": { "isAllowed": false, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": -1 } + }, + + { + "name": "table: finance.invoices, user: res-user, access: select", + "request": { + "resource": { "elements": { "database": "finance", "table": "invoices" } }, + "accessType": "select", "user": "res-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 110 } + }, + { + "name": "table: finance.invoices, user: tag-user, access: select", + "request": { + "resource": { "elements": { "database": "finance", "table": "invoices" } }, + "accessType": "select", "user": "tag-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 201 } + }, + { + "name": "table: finance.invoices, user: ds-user, access: select", + "request": { + "resource": { "elements": { "database": "finance", "table": "invoices" } }, + "accessType": "select", "user": "ds-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 2001 } + }, + { + "name": "table: finance.invoices, user: ds1-user, access: select", + "request": { + "resource": { "elements": { "database": "finance", "table": "invoices" } }, + "accessType": "select", "user": "ds1-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 2001 } + }, + { + "name": "table: finance.invoices, user: proj-user, access: select", + "request": { + "resource": { "elements": { "database": "finance", "table": "invoices" } }, + "accessType": "select", "user": "proj-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 3001 } + }, + { + "name": "table: finance.invoices, user: proj1-user, access: select", + "request": { + "resource": { "elements": { "database": "finance", "table": "invoices" } }, + "accessType": "select", "user": "proj1-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 3001 } + }, + { + "name": "table: finance.invoices, user: no-user, access: select", + "request": { + "resource": { "elements": { "database": "finance", "table": "invoices" } }, + "accessType": "select", "user": "no-user", "userGroups": [] + }, + "result": { "isAllowed": false, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": -1 } + }, + + { + "name": "table: shipping.shipments, user: res-user, access: select", + "request": { + "resource": { "elements": { "database": "shipping", "table": "shipments" } }, + "accessType": "select", "user": "res-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 121 } + }, + { + "name": "table: shipping.shipments, user: tag-user, access: select", + "request": { + "resource": { "elements": { "database": "shipping", "table": "shipments" } }, + "accessType": "select", "user": "tag-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 202 } + }, + { + "name": "table: shipping.shipments, user: ds-user, access: select", + "request": { + "resource": { "elements": { "database": "shipping", "table": "shipments" } }, + "accessType": "select", "user": "ds-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 2002 } + }, + { + "name": "table: shipping.shipments, user: ds2-user, access: select", + "request": { + "resource": { "elements": { "database": "shipping", "table": "shipments" } }, + "accessType": "select", "user": "ds2-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 2002 } + }, + { + "name": "table: shipping.shipments, user: proj-user, access: select", + "request": { + "resource": { "elements": { "database": "shipping", "table": "shipments" } }, + "accessType": "select", "user": "proj-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 3001 } + }, + { + "name": "table: shipping.shipments, user: proj1-user, access: select", + "request": { + "resource": { "elements": { "database": "shipping", "table": "shipments" } }, + "accessType": "select", "user": "proj1-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 3001 } + }, + { + "name": "table: shipping.shipments, user: no-user, access: select", + "request": { + "resource": { "elements": { "database": "shipping", "table": "shipments" } }, + "accessType": "select", "user": "no-user", "userGroups": [] + }, + "result": { "isAllowed": false, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": -1 } + }, + + + { + "name": "table: customers.contact_info, user: res-user, access: select", + "request": { + "resource": { "elements": { "database": "customers", "table": "contact_info" } }, + "accessType": "select", "user": "res-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 131 } + }, + { + "name": "table: customers.contact_info, user: tag-user, access: select", + "request": { + "resource": { "elements": { "database": "customers", "table": "contact_info" } }, + "accessType": "select", "user": "tag-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 203 } + }, + { + "name": "table: customers.contact_info, user: ds-user, access: select", + "request": { + "resource": { "elements": { "database": "customers", "table": "contact_info" } }, + "accessType": "select", "user": "ds-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 2003 } + }, + { + "name": "table: customers.contact_info, user: ds3-user, access: select", + "request": { + "resource": { "elements": { "database": "customers", "table": "contact_info" } }, + "accessType": "select", "user": "ds3-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 2003 } + }, + { + "name": "table: customers.contact_info, user: proj-user, access: select", + "request": { + "resource": { "elements": { "database": "customers", "table": "contact_info" } }, + "accessType": "select", "user": "proj-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 3002 } + }, + { + "name": "table: customers.contact_info, user: proj2-user, access: select", + "request": { + "resource": { "elements": { "database": "customers", "table": "contact_info" } }, + "accessType": "select", "user": "proj2-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 3002 } + }, + { + "name": "table: customers.contact_info, user: no-user, access: select", + "request": { + "resource": { "elements": { "database": "customers", "table": "contact_info" } }, + "accessType": "select", "user": "no-user", "userGroups": [] + }, + "result": { "isAllowed": false, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": -1 } + }, + + { + "name": "table: operations.facilities, user: res-user, access: select", + "request": { + "resource": { "elements": { "database": "operations", "table": "facilities" } }, + "accessType": "select", "user": "res-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 141 } + }, + { + "name": "table: operations.facilities, user: tag-user, access: select", + "request": { + "resource": { "elements": { "database": "operations", "table": "facilities" } }, + "accessType": "select", "user": "tag-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 204 } + }, + { + "name": "table: operations.facilities, user: ds-user, access: select", + "request": { + "resource": { "elements": { "database": "operations", "table": "facilities" } }, + "accessType": "select", "user": "ds-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 2004 } + }, + { + "name": "table: operations.facilities, user: ds4-user, access: select", + "request": { + "resource": { "elements": { "database": "operations", "table": "facilities" } }, + "accessType": "select", "user": "ds4-user", "userGroups": [] + }, + "result": { "isAllowed": true, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": 2004 } + }, + { + "name": "table: operations.facilities, user: proj-user, access: select", + "request": { + "resource": { "elements": { "database": "operations", "table": "facilities" } }, + "accessType": "select", "user": "proj-user", "userGroups": [] + }, + "result": { "isAllowed": false, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": -1 } + }, + { + "name": "table: operations.facilities, user: no-user, access: select", + "request": { + "resource": { "elements": { "database": "operations", "table": "facilities" } }, + "accessType": "select", "user": "no-user", "userGroups": [] + }, + "result": { "isAllowed": false, "isAccessDetermined": "true", "isAudited": true, "isAuditedDetermined": true, "policyId": -1 } + }, + + { + "name": "ACLs: table: sales.prospects", + "request": { + "resource": { "elements": { "database": "sales", "table": "prospects" } } + }, + "acls": { + "userACLs": { + "res-user": { "select": { "result": 1, "isFinal": true, "policy": { "id": 100 } } }, + "tag-user": { "select": { "result": 1, "isFinal": true, "policy": { "id": 200 } } }, + "ds-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 2001 } } }, + "ds1-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 2001 } } }, + "proj-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 3001 } } }, + "proj1-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 3001 } } } + }, + "datasets": [ "dataset-1" ], + "projects": [ "project-1" ] + } + }, + { + "name": "ACLs: table: finance.invoices", + "request": { + "resource": { "elements": { "database": "finance", "table": "invoices" } } + }, + "acls": { + "userACLs": { + "res-user": { "select": { "result": 1, "isFinal": true, "policy": { "id": 100 } } }, + "tag-user": { "select": { "result": 1, "isFinal": true, "policy": { "id": 200 } } }, + "ds-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 2001 } } }, + "ds1-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 2001 } } }, + "ds2-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 2002 } } }, + "proj-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 3001 } } }, + "proj1-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 3001 } } } + }, + "datasets": [ "dataset-1", "dataset-2" ], + "projects": [ "project-1" ] + } + }, + { + "name": "ACLs: table: shipping.shipments", + "request": { + "resource": { "elements": { "database": "shipping", "table": "shipments" } } + }, + "acls": { + "userACLs": { + "res-user": { "select": { "result": 1, "isFinal": true, "policy": { "id": 100 } } }, + "tag-user": { "select": { "result": 1, "isFinal": true, "policy": { "id": 200 } } }, + "ds-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 2002 } } }, + "ds2-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 2002 } } }, + "proj-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 3001 } } }, + "proj1-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 3001 } } } + }, + "datasets": [ "dataset-2" ], + "projects": [ "project-1" ] + } + }, + { + "name": "ACLs: table: customers.contact_info", + "request": { + "resource": { "elements": { "database": "customers", "table": "contact_info" } } + }, + "acls": { + "userACLs": { + "res-user": { "select": { "result": 1, "isFinal": true, "policy": { "id": 131 } } }, + "tag-user": { "select": { "result": 1, "isFinal": true, "policy": { "id": 203 } } }, + "ds-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 2003 } } }, + "ds3-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 2003 } } }, + "proj-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 3002 } } }, + "proj2-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 3002 } } } + }, + "datasets": [ "dataset-3" ], + "projects": [ "project-2" ] + } + }, + { + "name": "ACLs: table: operations.facilities", + "request": { + "resource": { "elements": { "database": "operations", "table": "facilities" } } + }, + "acls": { + "userACLs": { + "res-user": { "select": { "result": 1, "isFinal": true, "policy": { "id": 141 } } }, + "tag-user": { "select": { "result": 1, "isFinal": true, "policy": { "id": 204 } } }, + "ds-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 2004 } } }, + "ds4-user": { "select": { "result": 1, "isFinal": false, "policy": { "id": 2004 } } } + }, + "datasets": [ "dataset-4" ] + } + } + ] +} \ No newline at end of file diff --git a/agents-common/src/test/resources/policyengine/gds/gds_info_hive.json b/agents-common/src/test/resources/policyengine/gds/gds_info_hive.json new file mode 100644 index 000000000..34fcc6290 --- /dev/null +++ b/agents-common/src/test/resources/policyengine/gds/gds_info_hive.json @@ -0,0 +1,153 @@ +{ + "serviceName": "dev_hive", + "dataShares": [ + { "id": 1, "name": "hive-sales-2023", "zoneName": "sales", "conditionExpr": "", "defaultAccessTypes": [ "_READ" ], "defaultTagMasks": [ ] }, + { "id": 2, "name": "hive-finance-2023", "zoneName": "finance", "conditionExpr": "", "defaultAccessTypes": [ "_READ" ], "defaultTagMasks": [ ] }, + { "id": 3, "name": "hive-shipping-2023", "zoneName": "shipping", "conditionExpr": "", "defaultAccessTypes": [ "_READ" ], "defaultTagMasks": [ ] }, + { "id": 4, "name": "hive-new-customers-2023", "zoneName": "", "conditionExpr": "", "defaultAccessTypes": [ "_READ" ], "defaultTagMasks": [ ] }, + { "id": 5, "name": "hive-facilities", "zoneName": "", "conditionExpr": "", "defaultAccessTypes": [ "_READ" ], "defaultTagMasks": [ ] } + ], + "dshids": [ + { "dataShareId": 1, "datasetId": 1, "status": "ACTIVE", "validitySchedule": null }, + { "dataShareId": 2, "datasetId": 1, "status": "ACTIVE", "validitySchedule": null }, + { "dataShareId": 2, "datasetId": 2, "status": "ACTIVE", "validitySchedule": null }, + { "dataShareId": 3, "datasetId": 2, "status": "ACTIVE", "validitySchedule": null }, + { "dataShareId": 4, "datasetId": 3, "status": "ACTIVE", "validitySchedule": null }, + { "dataShareId": 5, "datasetId": 4, "status": "ACTIVE", "validitySchedule": null } + ], + "dips": [ + { "datasetId": 1, "projectId": 1, "status": "ACTIVE", "validitySchedule": null }, + { "datasetId": 2, "projectId": 1, "status": "ACTIVE", "validitySchedule": null }, + { "datasetId": 3, "projectId": 2, "status": "ACTIVE", "validitySchedule": null } + ], + "resources": [ + { + "id": 11, "dataShareId": 1, "conditionExpr": "", "accessTypes": [ "select" ], + "resource": { "database": { "values": [ "sales" ] }, "table": { "values": [ "prospects" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, + "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": { } + }, + { + "id": 12, "dataShareId": 1, "conditionExpr": "", "accessTypes": [ "select" ], + "resource": { "database": { "values": [ "sales" ] }, "table": { "values": [ "orders" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, + "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": { } + }, + { + "id": 21, "dataShareId": 2, "conditionExpr": "", "accessTypes": [ "select" ], + "resource": { "database": { "values": [ "finance" ] }, "table": { "values": [ "invoices" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, + "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": { } + }, + { + "id": 22, "dataShareId": 2, "conditionExpr": "", "accessTypes": [ "select" ], + "resource": { "database": { "values": [ "finance" ] }, "table": { "values": [ "payments" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, + "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": { } + }, + { + "id": 31, "dataShareId": 3, "conditionExpr": "", "accessTypes": [ "select" ], + "resource": { "database": { "values": [ "shipping" ] }, "table": { "values": [ "shipments" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, + "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": { } + }, + { + "id": 41, "dataShareId": 4, "conditionExpr": "", "accessTypes": [ "select" ], + "resource": { "database": { "values": [ "customers" ] }, "table": { "values": [ "contact_info" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, + "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": null + }, + { + "id": 51, "dataShareId": 5, "conditionExpr": "", "accessTypes": [ "select" ], + "resource": { "database": { "values": [ "operations" ] }, "table": { "values": [ "facilities" ] } }, "rowFilter": null, + "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": null + } + ], + "datasets": [ + { + "id": 1, + "name": "dataset-1", + "policies": [ + { "id": 2001, "name": "dataset-1", "isEnabled": true, "isAuditEnabled": true, + "resources": { "dataset-id": { "values": ["1"] } }, + "policyItems":[ + { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "ds-user" ], "groups": []} + ] + } + ] + }, + { + "id": 2, + "name": "dataset-2", + "policies": [ + { "id": 2002, "name": "dataset-2", "isEnabled": true, "isAuditEnabled": true, + "resources": { "dataset-id": { "values": ["2"] } }, + "policyItems":[ + { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "ds-user" ], "groups": []} + ] + } + ] + }, + { + "id": 3, + "name": "dataset-3", + "policies": [ + { "id": 2003, "name": "dataset-3", "isEnabled": true, "isAuditEnabled": true, + "resources": { "dataset-id": { "values": ["3"] } }, + "policyItems":[ + { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "ds-user" ], "groups": []} + ] + } + ] + }, + { + "id": 4, + "name": "dataset-4", + "policies": [ + { "id": 2004, "name": "dataset-4", "isEnabled": true, "isAuditEnabled": true, + "resources": { "dataset-id": { "values": ["4"] } }, + "policyItems":[ + { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "ds-user" ], "groups": []} + ] + } + ] + } + ], + "projects": [ + { + "id": 1, + "name": "project-1", + "policies": [ + { "id": 3001, "name": "project-1", "isEnabled": true, "isAuditEnabled": true, + "resources": { "project-id": { "values": ["1"] } }, + "policyItems":[ + { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "proj-user" ], "groups": []} + ] + } + ] + }, + { + "id": 2, + "name": "project-2", + "policies": [ + { "id": 3002, "name": "project-2", "isEnabled": true, "isAuditEnabled": true, + "resources": { "project-id": { "values": ["2"] } }, + "policyItems":[ + { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "proj-user" ], "groups": []} + ] + } + ] + } + ], + "gdsServiceDef": { + "name": "gds", + "id": 3, + "resources": [ + { "name": "dataset-id", "level":1, "parent": "", "mandatory": true, "lookupSupported": false, "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", "matcherOptions":{ "wildCard": false, "ignoreCase": true }, "label": "Dataset ID", "description": "Dataset ID" }, + { "name": "project-id", "level":1, "parent": "", "mandatory": true, "lookupSupported": false, "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", "matcherOptions":{ "wildCard": false, "ignoreCase": true }, "label": "Project ID", "description": "Project ID" } + ], + "accessTypes":[ + { "itemId": 1, "name": "_CREATE", "label": "_CREATE" }, + { "itemId": 2, "name": "_READ", "label": "_READ" }, + { "itemId": 3, "name": "_UPDATE", "label": "_UPDATE" }, + { "itemId": 4, "name": "_DELETE", "label": "_DELETE" }, + { "itemId": 5, "name": "_MANAGE", "label": "_MANAGE" }, + { "itemId": 6, "name": "_ALL", "label": "_ALL" } + ] + }, + "gdsVersion": 1 +} \ No newline at end of file diff --git a/agents-common/src/test/resources/policyengine/gds/test_gds_policy_engine_hive.json b/agents-common/src/test/resources/policyengine/gds/test_gds_policy_engine_hive.json index 89fed5f9e..2ab93bde6 100644 --- a/agents-common/src/test/resources/policyengine/gds/test_gds_policy_engine_hive.json +++ b/agents-common/src/test/resources/policyengine/gds/test_gds_policy_engine_hive.json @@ -32,159 +32,7 @@ "finance": { "zoneName": "finance", "resources": [ { "database": [ "finance" ] } ] }, "shipping": { "zoneName": "shipping", "resources": [ { "database": [ "shipping" ] } ] } }, - "gdsInfo": { - "serviceName": "dev_hive", - "dataShares": [ - { "id": 1, "name": "hive-sales-2023", "zoneName": "sales", "conditionExpr": "", "defaultAccessTypes": [ "_READ" ], "defaultTagMasks": [ ] }, - { "id": 2, "name": "hive-finance-2023", "zoneName": "finance", "conditionExpr": "", "defaultAccessTypes": [ "_READ" ], "defaultTagMasks": [ ] }, - { "id": 3, "name": "hive-shipping-2023", "zoneName": "shipping", "conditionExpr": "", "defaultAccessTypes": [ "_READ" ], "defaultTagMasks": [ ] }, - { "id": 4, "name": "hive-new-customers-2023", "zoneName": "", "conditionExpr": "", "defaultAccessTypes": [ "_READ" ], "defaultTagMasks": [ ] }, - { "id": 5, "name": "hive-facilities", "zoneName": "", "conditionExpr": "", "defaultAccessTypes": [ "_READ" ], "defaultTagMasks": [ ] } - ], - "dshids": [ - { "dataShareId": 1, "datasetId": 1, "status": "ACTIVE", "validitySchedule": null }, - { "dataShareId": 2, "datasetId": 1, "status": "ACTIVE", "validitySchedule": null }, - { "dataShareId": 2, "datasetId": 2, "status": "ACTIVE", "validitySchedule": null }, - { "dataShareId": 3, "datasetId": 2, "status": "ACTIVE", "validitySchedule": null }, - { "dataShareId": 4, "datasetId": 3, "status": "ACTIVE", "validitySchedule": null }, - { "dataShareId": 5, "datasetId": 4, "status": "ACTIVE", "validitySchedule": null } - ], - "dips": [ - { "datasetId": 1, "projectId": 1, "status": "ACTIVE", "validitySchedule": null }, - { "datasetId": 2, "projectId": 1, "status": "ACTIVE", "validitySchedule": null }, - { "datasetId": 3, "projectId": 2, "status": "ACTIVE", "validitySchedule": null } - ], - "resources": [ - { - "id": 11, "dataShareId": 1, "conditionExpr": "", "accessTypes": [ "select" ], - "resource": { "database": { "values": [ "sales" ] }, "table": { "values": [ "prospects" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, - "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": { } - }, - { - "id": 12, "dataShareId": 1, "conditionExpr": "", "accessTypes": [ "select" ], - "resource": { "database": { "values": [ "sales" ] }, "table": { "values": [ "orders" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, - "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": { } - }, - { - "id": 21, "dataShareId": 2, "conditionExpr": "", "accessTypes": [ "select" ], - "resource": { "database": { "values": [ "finance" ] }, "table": { "values": [ "invoices" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, - "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": { } - }, - { - "id": 22, "dataShareId": 2, "conditionExpr": "", "accessTypes": [ "select" ], - "resource": { "database": { "values": [ "finance" ] }, "table": { "values": [ "payments" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, - "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": { } - }, - { - "id": 31, "dataShareId": 3, "conditionExpr": "", "accessTypes": [ "select" ], - "resource": { "database": { "values": [ "shipping" ] }, "table": { "values": [ "shipments" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, - "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": { } - }, - { - "id": 41, "dataShareId": 4, "conditionExpr": "", "accessTypes": [ "select" ], - "resource": { "database": { "values": [ "customers" ] }, "table": { "values": [ "contact_info" ] } }, "rowFilter": { "filterExpr": "created_time >= '2023-01-01' and created_time < '2024-01-01'" }, - "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": null - }, - { - "id": 51, "dataShareId": 5, "conditionExpr": "", "accessTypes": [ "select" ], - "resource": { "database": { "values": [ "operations" ] }, "table": { "values": [ "facilities" ] } }, "rowFilter": null, - "subResourceType": "column", "subResource": { "values": [ "*" ] }, "subResourceMasks": null - } - ], - "datasets": [ - { - "id": 1, - "name": "dataset-1", - "policies": [ - { "id": 2001, "name": "dataset-1", "isEnabled": true, "isAuditEnabled": true, - "resources": { "dataset-id": { "values": ["1"] } }, - "policyItems":[ - { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "ds-user" ], "groups": []} - ] - } - ] - }, - { - "id": 2, - "name": "dataset-2", - "policies": [ - { "id": 2002, "name": "dataset-2", "isEnabled": true, "isAuditEnabled": true, - "resources": { "dataset-id": { "values": ["2"] } }, - "policyItems":[ - { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "ds-user" ], "groups": []} - ] - } - ] - }, - { - "id": 3, - "name": "dataset-3", - "policies": [ - { "id": 2003, "name": "dataset-3", "isEnabled": true, "isAuditEnabled": true, - "resources": { "dataset-id": { "values": ["3"] } }, - "policyItems":[ - { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "ds-user" ], "groups": []} - ] - } - ] - }, - { - "id": 4, - "name": "dataset-4", - "policies": [ - { "id": 2004, "name": "dataset-4", "isEnabled": true, "isAuditEnabled": true, - "resources": { "dataset-id": { "values": ["4"] } }, - "policyItems":[ - { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "ds-user" ], "groups": []} - ] - } - ] - } - ], - "projects": [ - { - "id": 1, - "name": "project-1", - "policies": [ - { "id": 3001, "name": "project-1", "isEnabled": true, "isAuditEnabled": true, - "resources": { "project-id": { "values": ["1"] } }, - "policyItems":[ - { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "proj-user" ], "groups": []} - ] - } - ] - }, - { - "id": 2, - "name": "project-2", - "policies": [ - { "id": 3002, "name": "project-2", "isEnabled": true, "isAuditEnabled": true, - "resources": { "project-id": { "values": ["2"] } }, - "policyItems":[ - { "accesses":[ { "type": "_ALL", "isAllowed": true } ], "users": [ "proj-user" ], "groups": []} - ] - } - ] - } - ], - "gdsServiceDef": { - "name": "gds", - "id": 3, - "resources": [ - { "name": "dataset-id", "level":1, "parent": "", "mandatory": true, "lookupSupported": false, "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", "matcherOptions":{ "wildCard": false, "ignoreCase": true }, "label": "Dataset ID", "description": "Dataset ID" }, - { "name": "project-id", "level":1, "parent": "", "mandatory": true, "lookupSupported": false, "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", "matcherOptions":{ "wildCard": false, "ignoreCase": true }, "label": "Project ID", "description": "Project ID" } - ], - "accessTypes":[ - { "itemId": 1, "name": "_CREATE", "label": "_CREATE" }, - { "itemId": 2, "name": "_READ", "label": "_READ" }, - { "itemId": 3, "name": "_UPDATE", "label": "_UPDATE" }, - { "itemId": 4, "name": "_DELETE", "label": "_DELETE" }, - { "itemId": 5, "name": "_MANAGE", "label": "_MANAGE" }, - { "itemId": 6, "name": "_ALL", "label": "_ALL" } - ] - }, - "gdsVersion": 1 - }, + "gdsInfoFilename": "/policyengine/gds/gds_info_hive.json", "tests": [ { "name": "table: sales.prospects, user: ds-user, access: select", @@ -388,8 +236,8 @@ "request": { "resource": { "elements": { "database": "sales", "table": "prospects" } } }, "acls": { "userACLs": { - "ds-user": { "select": { "result": 1, "isFinal": false } }, - "proj-user": { "select": { "result": 1, "isFinal": false } } + "ds-user": { "select": { "result": 1, "isFinal": true } }, + "proj-user": { "select": { "result": 1, "isFinal": true } } }, "datasets": [ "dataset-1" ], "projects": [ "project-1" ] @@ -400,8 +248,8 @@ "request": { "resource": { "elements": { "database": "sales", "table": "orders" } } }, "acls": { "userACLs": { - "ds-user": { "select": { "result": 1, "isFinal": false } }, - "proj-user": { "select": { "result": 1, "isFinal": false } } + "ds-user": { "select": { "result": 1, "isFinal": true } }, + "proj-user": { "select": { "result": 1, "isFinal": true } } }, "datasets": [ "dataset-1" ], "projects": [ "project-1" ] @@ -417,8 +265,8 @@ "request": { "resource": { "elements": { "database": "sales", "table": "orders", "column": "created_time" } } }, "acls": { "userACLs": { - "ds-user": { "select": { "result": 1, "isFinal": false } }, - "proj-user": { "select": { "result": 1, "isFinal": false } } + "ds-user": { "select": { "result": 1, "isFinal": true } }, + "proj-user": { "select": { "result": 1, "isFinal": true } } }, "datasets": [ "dataset-1" ], "projects": [ "project-1" ] @@ -434,8 +282,8 @@ "request": { "resource": { "elements": { "database": "finance", "table": "invoices" } } }, "acls": { "userACLs": { - "ds-user": { "select": { "result": 1, "isFinal": false } }, - "proj-user": { "select": { "result": 1, "isFinal": false } } + "ds-user": { "select": { "result": 1, "isFinal": true } }, + "proj-user": { "select": { "result": 1, "isFinal": true } } }, "datasets": [ "dataset-1", "dataset-2" ], "projects": [ "project-1" ] @@ -446,8 +294,8 @@ "request": { "resource": { "elements": { "database": "finance", "table": "payments" } } }, "acls": { "userACLs": { - "ds-user": { "select": { "result": 1, "isFinal": false } }, - "proj-user": { "select": { "result": 1, "isFinal": false } } + "ds-user": { "select": { "result": 1, "isFinal": true } }, + "proj-user": { "select": { "result": 1, "isFinal": true } } }, "datasets": [ "dataset-1", "dataset-2" ], "projects": [ "project-1" ] @@ -463,8 +311,8 @@ "request": { "resource": { "elements": { "database": "shipping", "table": "shipments" } } }, "acls": { "userACLs": { - "ds-user": { "select": { "result": 1, "isFinal": false } }, - "proj-user": { "select": { "result": 1, "isFinal": false } } + "ds-user": { "select": { "result": 1, "isFinal": true } }, + "proj-user": { "select": { "result": 1, "isFinal": true } } }, "datasets": [ "dataset-2" ], "projects": [ "project-1" ] @@ -480,8 +328,8 @@ "request": { "resource": { "elements": { "database": "customers", "table": "contact_info" } } }, "acls": { "userACLs": { - "ds-user": { "select": { "result": 1, "isFinal": false } }, - "proj-user": { "select": { "result": 1, "isFinal": false } } + "ds-user": { "select": { "result": 1, "isFinal": true } }, + "proj-user": { "select": { "result": 1, "isFinal": true } } }, "datasets": [ "dataset-3" ], "projects": [ "project-2" ] @@ -497,7 +345,7 @@ "request": { "resource": { "elements": { "database": "operations", "table": "facilities" } } }, "acls": { "userACLs": { - "ds-user": { "select": { "result": 1, "isFinal": false } } + "ds-user": { "select": { "result": 1, "isFinal": true } } }, "datasets": [ "dataset-4" ] } diff --git a/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java b/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java index 5b6f51339..4c01d4185 100644 --- a/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java +++ b/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java @@ -30,10 +30,11 @@ import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig; import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; import org.apache.ranger.plugin.service.RangerBasePlugin; import org.apache.ranger.plugin.util.RangerRoles; +import org.apache.ranger.plugin.util.RangerUserStore; +import org.apache.ranger.plugin.util.ServiceGdsInfo; import org.apache.ranger.plugin.util.ServicePolicies; import org.apache.ranger.plugin.util.ServicePolicies.SecurityZoneInfo; import org.apache.ranger.plugin.util.ServiceTags; -import org.apache.ranger.plugin.util.RangerUserStore; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -49,6 +50,7 @@ public class RangerMemSizing { private final String tagFile; private final String rolesFile; private final String userStoreFile; + private final String gdsInfoFile; private final boolean deDup; private final boolean deDupStrings; private final String optimizationMode; @@ -61,6 +63,7 @@ public class RangerMemSizing { this.tagFile = cmdLine.getOptionValue('t'); this.rolesFile = cmdLine.getOptionValue('r'); this.userStoreFile = cmdLine.getOptionValue('u'); + this.gdsInfoFile = cmdLine.getOptionValue('g'); this.deDup = Boolean.parseBoolean(cmdLine.getOptionValue("d", "true")); this.deDupStrings = this.deDup; this.optimizationMode = StringUtils.startsWithIgnoreCase(cmdLine.getOptionValue('o', "space"), "s") ? OPT_MODE_SPACE : OPT_MODEL_RETRIEVAL; @@ -73,7 +76,8 @@ public class RangerMemSizing { ServiceTags tags = loadTags(tagFile, tracker); RangerRoles roles = loadRoles(rolesFile, tracker); RangerUserStore userStore = loadUserStore(userStoreFile, tracker); - RangerBasePlugin plugin = createRangerPlugin(policies, tags, roles, userStore, tracker); + ServiceGdsInfo gdsInfo = loadGdsInfo(gdsInfoFile, tracker); + RangerBasePlugin plugin = createRangerPlugin(policies, tags, roles, userStore, gdsInfo, tracker); tracker.stop(); @@ -293,7 +297,53 @@ public class RangerMemSizing { return ret; } - private RangerBasePlugin createRangerPlugin(ServicePolicies policies, ServiceTags tags, RangerRoles roles, RangerUserStore userStore, PerfMemTimeTracker parent) { + private ServiceGdsInfo loadGdsInfo(String fileName, PerfMemTimeTracker parent) { + if (fileName == null) { + return null; + } + + ServiceGdsInfo ret = null; + + try { + File file = new File(fileName); + PerfMemTimeTracker loadTracker = new PerfMemTimeTracker("Load gdsInfo"); + + log("loading gdsInfo(file=" + fileName + ")"); + + { + PerfMemTimeTracker tracker = new PerfMemTimeTracker("Read gdsInfo"); + + try (FileReader reader = new FileReader(file)) { + ret = gson.fromJson(reader, ServiceGdsInfo.class); + } + + tracker.stop(); + loadTracker.addChild(tracker); + } + + if (deDupStrings) { + PerfMemTimeTracker tracker = new PerfMemTimeTracker("DeDupStrings"); + + ret.dedupStrings(); + + tracker.stop(); + loadTracker.addChild(tracker); + } + + loadTracker.stop(); + parent.addChild(loadTracker); + + log("loaded gdsInfo(file=" + fileName + ", size=" + file.length() + "): " + toSummaryStr(ret) + ")"); + } catch (FileNotFoundException excp) { + log(fileName + ": file does not exist!"); + } catch (IOException excp) { + log(fileName, excp); + } + + return ret; + } + + private RangerBasePlugin createRangerPlugin(ServicePolicies policies, ServiceTags tags, RangerRoles roles, RangerUserStore userStore, ServiceGdsInfo gdsInfo, PerfMemTimeTracker parent) { RangerBasePlugin ret = null; if (policies != null) { @@ -305,7 +355,7 @@ public class RangerMemSizing { log("Initializing RangerBasePlugin..."); - ret = new RangerBasePlugin(pluginConfig, policies, tags, roles, userStore); + ret = new RangerBasePlugin(pluginConfig, policies, tags, roles, userStore, gdsInfo); tracker.stop(); parent.addChild(tracker); @@ -467,4 +517,31 @@ public class RangerMemSizing { return "users=" + userCount + ", groups=" + groupCount + ", userGroupMappings=" + userGroupCount; } + + private static String toSummaryStr(ServiceGdsInfo gdsInfo) { + int dataShareCount = 0; + int resourcesCount = 0; + int datasetCount = 0; + int projectCount = 0; + + if (gdsInfo != null) { + if (gdsInfo.getDataShares() != null) { + dataShareCount = gdsInfo.getDataShares().size(); + } + + if (gdsInfo.getResources() != null) { + resourcesCount = gdsInfo.getResources().size(); + } + + if (gdsInfo.getDatasets() != null) { + datasetCount = gdsInfo.getDatasets().size(); + } + + if (gdsInfo.getProjects() != null) { + projectCount = gdsInfo.getProjects().size(); + } + } + + return "dataShares=" + dataShareCount + ", resources=" + resourcesCount + ", datasets=" + datasetCount + ", projects=" + projectCount; + } }