Repository: incubator-ranger Updated Branches: refs/heads/tag-policy 3a0582d89 -> e25baa775
RANGER-274: TagFileStore based context-enricher for tag serviceDef Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/e25baa77 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/e25baa77 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/e25baa77 Branch: refs/heads/tag-policy Commit: e25baa775a0322baa436fdb6215ae73563a0d33b Parents: 3a0582d Author: Abhay Kulkarni <[email protected]> Authored: Fri May 29 10:58:34 2015 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Thu Jun 4 18:51:45 2015 -0700 ---------------------------------------------------------------------- .../RangerScriptConditionEvaluator.java | 77 +++--- .../RangerScriptExecutionContext.java | 243 +++++++++++++++++++ .../RangerAbstractContextEnricher.java | 25 +- .../contextenricher/RangerContextEnricher.java | 4 +- .../RangerTagFileStoreRetriever.java | 77 ++++++ .../contextenricher/RangerTagProvider.java | 164 +++++++++++++ .../contextenricher/RangerTagReceiver.java | 28 +++ .../contextenricher/RangerTagRefresher.java | 110 +++++++++ .../contextenricher/RangerTagRetriever.java | 31 +++ .../RangerTaggedResourceMatcher.java | 38 +++ .../RangerAccessResourceReadOnly.java | 21 +- .../policyengine/RangerPolicyEngineImpl.java | 23 +- .../policyengine/RangerPolicyRepository.java | 163 ++++++++++--- .../plugin/store/AbstractServiceStore.java | 13 +- .../ranger/plugin/store/AbstractTagStore.java | 7 - .../ranger/plugin/store/TagPredicateUtil.java | 4 +- .../apache/ranger/plugin/store/TagStore.java | 5 +- .../ranger/plugin/store/file/TagFileStore.java | 10 +- .../apache/ranger/plugin/util/SearchFilter.java | 2 +- .../service-defs/ranger-servicedef-_tag_.json | 19 +- .../policyengine/test_policyengine_hdfs.json | 38 ++- .../java/org/apache/ranger/rest/TagREST.java | 8 +- .../apache/ranger/rest/TagRESTConstants.java | 3 +- 23 files changed, 972 insertions(+), 141 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java index 558e35e..800c0b7 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java @@ -24,14 +24,12 @@ import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.ranger.plugin.model.RangerResource; import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import javax.script.Bindings; import javax.script.ScriptEngine; import javax.script.ScriptEngineManager; import javax.script.ScriptException; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -61,8 +59,16 @@ public class RangerScriptConditionEvaluator extends RangerAbstractConditionEvalu engineName = "JavaScript"; } - ScriptEngineManager manager = new ScriptEngineManager(); - scriptEngine = manager.getEngineByName(engineName); + if (LOG.isDebugEnabled()) { + LOG.debug("RangerScriptConditionEvaluator.init() - engineName=" + engineName); + } + + try { + ScriptEngineManager manager = new ScriptEngineManager(); + scriptEngine = manager.getEngineByName(engineName); + } catch (Exception exp) { + LOG.error("RangerScriptConditionEvaluator.init() failed with exception=" + exp); + } if (LOG.isDebugEnabled()) { LOG.debug("<== RangerScriptConditionEvaluator.init(" + condition + ")"); @@ -74,55 +80,48 @@ public class RangerScriptConditionEvaluator extends RangerAbstractConditionEvalu if (LOG.isDebugEnabled()) { LOG.debug("==>RangerScriptConditionEvaluator.isMatched()"); } + boolean result = false; - Boolean result = false; - - List<String> values = condition.getValues(); + if (scriptEngine != null) { - if (!CollectionUtils.isEmpty(values)) { + List<String> values = condition.getValues(); - // Evaluate the first string - String value = values.get(0); - if (StringUtils.isNotBlank(value)) { + if (CollectionUtils.isNotEmpty(values)) { - RangerAccessRequest readOnlyRequest = request.getReadOnlyCopy(); + String value = values.get(0); + if (StringUtils.isNotBlank(value)) { - @SuppressWarnings("unchecked") - List<RangerResource.RangerResourceTag> tagsList = (List <RangerResource.RangerResourceTag>)readOnlyRequest.getContext().get("TAGS"); + RangerAccessRequest readOnlyRequest = request.getReadOnlyCopy(); - Bindings bindings = scriptEngine.createBindings(); + RangerScriptExecutionContext context = new RangerScriptExecutionContext(readOnlyRequest); - if (CollectionUtils.isNotEmpty(tagsList)) { - List<RangerResource.RangerResourceTag> readOnlyTags = Collections.unmodifiableList(tagsList); - bindings.put("tags", readOnlyTags); - } + Bindings bindings = scriptEngine.createBindings(); - bindings.put("request", readOnlyRequest); - bindings.put("ctx", readOnlyRequest.getContext()); - bindings.put("result", result); + bindings.put("ctx", context); - String script = value.trim(); + String script = value.trim(); - if (LOG.isDebugEnabled()) { - LOG.debug("RangerScriptConditionEvaluator.isMatched(): script={" + script + "}"); - } - try { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerScriptConditionEvaluator.isMatched(): script={" + script + "}"); + } + try { - Object ret = scriptEngine.eval(script, bindings); + Object ret = scriptEngine.eval(script, bindings); - if (ret == null) { - ret = bindings.get("result"); - } - if (ret != null && ret instanceof Boolean) { - result = (Boolean) ret; - } + if (ret == null) { + ret = context.getResult(); + } + if (ret instanceof Boolean) { + result = (Boolean) ret; + } - } catch (NullPointerException nullp) { - LOG.error("RangerScriptConditionEvaluator.isMatched(): eval called with NULL argument(s)"); + } catch (NullPointerException nullp) { + LOG.error("RangerScriptConditionEvaluator.isMatched(): eval called with NULL argument(s)"); - } catch (ScriptException exception) { - LOG.error("RangerScriptConditionEvaluator.isMatched(): failed to evaluate script," + - " exception=" + exception); + } catch (ScriptException exception) { + LOG.error("RangerScriptConditionEvaluator.isMatched(): failed to evaluate script," + + " exception=" + exception); + } } } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java new file mode 100644 index 0000000..ff9387b --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptExecutionContext.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.conditionevaluator; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.model.RangerResource; +import org.apache.ranger.plugin.policyengine.RangerAccessRequest; +import org.apache.ranger.plugin.policyengine.RangerAccessResource; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; + +import java.text.SimpleDateFormat; +import java.util.*; + +public final class RangerScriptExecutionContext { + private static final Log LOG = LogFactory.getLog(RangerScriptExecutionContext.class); + public static final String DATETIME_FORMAT_PATTERN = "EEE MMM dd HH:mm:ss z yyyy"; + + private final RangerAccessRequest accessRequest; + private Boolean result = false; + + RangerScriptExecutionContext(final RangerAccessRequest accessRequest) { + this.accessRequest = accessRequest; + } + + public final String getResource() { + + @SuppressWarnings("unchecked") + RangerAccessResource resource = (RangerAccessResource)getEvaluationContext().get(RangerPolicyEngine.KEY_CONTEXT_RESOURCE); + + return resource != null ? resource.getAsString() : null; + } + + public final Map<String, Object> getEvaluationContext() { + return accessRequest.getContext(); + } + + public final boolean isAccessTypeAny() { return accessRequest.isAccessTypeAny(); } + + public final boolean isAccessTypeDelegatedAdmin() { return accessRequest.isAccessTypeDelegatedAdmin(); } + + public final String getUser() { return accessRequest.getUser(); } + + public final Set<String> getUserGroups() { return accessRequest.getUserGroups(); } + + public final Date getAccessTime() { return accessRequest.getAccessTime(); } + + public final String getClientIPAddress() { return accessRequest.getClientIPAddress(); } + + public final String getClientType() { return accessRequest.getClientType(); } + + public final String getAction() { return accessRequest.getAction(); } + + public final String getRequestData() { return accessRequest.getRequestData(); } + + public final String getSessionId() { return accessRequest.getSessionId(); } + + public final RangerResource.RangerResourceTag getCurrentTag() { + @SuppressWarnings("unchecked") + RangerResource.RangerResourceTag tagObject = (RangerResource.RangerResourceTag)getEvaluationContext() + .get(RangerPolicyEngine.KEY_CONTEXT_TAG_OBJECT); + if (tagObject == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("getCurrentTag() - No current TAG object. Script execution must be for resource-based policy."); + } + } + return tagObject; + } + + public final String getCurrentTagName() { + RangerResource.RangerResourceTag tagObject = getCurrentTag(); + return (tagObject != null) ? tagObject.getName() : null; + } + + public final Set<String> getAllTagNames() { + + Set<String> allTagNames = null; + + List<RangerResource.RangerResourceTag> tagObjectList = getAllTags(); + + if (CollectionUtils.isNotEmpty(tagObjectList)) { + + for (RangerResource.RangerResourceTag tag : tagObjectList) { + String tagName = tag.getName(); + if (allTagNames == null) { + allTagNames = new HashSet<String>(); + } + allTagNames.add(tagName); + } + } + + return allTagNames; + } + + public final Map<String, String> getTagAttributeValues(final String tagName) { + + Map<String, String> ret = null; + + if (StringUtils.isNotBlank(tagName)) { + + List<RangerResource.RangerResourceTag> tagObjectList = getAllTags(); + + // Assumption: There is exactly one tag with given tagName in the list of tags - may not be true ***TODO*** + // This will get attributeValues of the first tagName that matches + + if (CollectionUtils.isNotEmpty(tagObjectList)) { + for (RangerResource.RangerResourceTag tag : tagObjectList) { + if (tag.getName().equals(tagName)) { + ret = tag.getAttributeValues(); + break; + } + } + } + } + + return ret; + } + + public final Set<String> getAttributeNames(final String tagName) { + + Set<String> ret = null; + + if (StringUtils.isNotBlank(tagName)) { + Map<String, String> attributeValues = getTagAttributeValues(tagName); + + if (attributeValues != null) { + ret = attributeValues.keySet(); + } + } + return ret; + } + + public final String getAttributeValue(final String tagName, final String attributeName) { + + String ret = null; + Map<String, String> attributeValues; + + if (StringUtils.isNotBlank(tagName) || StringUtils.isNotBlank(attributeName)) { + attributeValues = getTagAttributeValues(tagName); + + if (attributeValues != null) { + ret = attributeValues.get(attributeName); + } + } + return ret; + } + + public final String getAttributeValue(final String attributeName) { + + String ret = null; + + if (StringUtils.isNotBlank(attributeName)) { + RangerResource.RangerResourceTag tag = getCurrentTag(); + Map<String, String> attributeValues = null; + if (tag != null) { + attributeValues = tag.getAttributeValues(); + } + if (attributeValues != null) { + ret = attributeValues.get(attributeName); + } + } + return ret; + } + + public final boolean getResult() { + return result; + + } + + public final void setResult(final boolean result) { + this.result = result; + } + + // Utilities - TODO + + public final Date getAsDate(String value) { + + Date ret = null; + + if (StringUtils.isNotBlank(value)) { + SimpleDateFormat df = new SimpleDateFormat(DATETIME_FORMAT_PATTERN); + try { + ret = df.parse(value); + } catch (Exception ex) { + LOG.error("RangerScriptExecutionContext.getAsDate() - Could not convert " + value + " to Date, exception=" + ex); + } + } + + return ret; + } + + public final Date getTagAttributeAsDate(String tagName, String attributeName) { + // sample JavaScript to demonstrate use of this helper method + + /* + + importPackage(java.util); + var expiryDate = ctx.getTagAttributeAsDate('PII', 'expiryDate') + var now = new Date(); + now.getTime() < expiryDate.getTime());" + + */ + + String attrValue = getAttributeValue(tagName, attributeName); + + return getAsDate(attrValue); + + } + + private List<RangerResource.RangerResourceTag> getAllTags() { + + @SuppressWarnings("unchecked") + List<RangerResource.RangerResourceTag> ret = (List<RangerResource.RangerResourceTag>)getEvaluationContext().get(RangerPolicyEngine.KEY_CONTEXT_TAGS); + + if (ret == null) { + if (LOG.isDebugEnabled()) { + String resource = accessRequest.getResource().getAsString(); + + LOG.debug("getAllTags() - No current TAGS. No TAGS for the RangerAccessResource=" + resource); + } + } + return ret; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java index 72affda..bd45ce6 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAbstractContextEnricher.java @@ -27,7 +27,6 @@ import java.util.Properties; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.model.RangerServiceDef.RangerContextEnricherDef; @@ -38,9 +37,6 @@ public abstract class RangerAbstractContextEnricher implements RangerContextEnri protected RangerContextEnricherDef enricherDef; protected String serviceName; protected RangerServiceDef serviceDef; - protected String componentServiceName; - protected RangerServiceDef componentServiceDef; - @Override public void setContextEnricherDef(RangerContextEnricherDef enricherDef) { @@ -68,16 +64,6 @@ public abstract class RangerAbstractContextEnricher implements RangerContextEnri this.serviceDef = serviceDef; } - @Override - public void setContextComponentServiceName(String componentServiceName) { - this.componentServiceName = componentServiceName; - } - - @Override - public void setContextComponentServiceDef(RangerServiceDef componentServiceDef) { - this.componentServiceDef = componentServiceDef; - } - public String getOption(String name) { String ret = null; @@ -123,6 +109,17 @@ public abstract class RangerAbstractContextEnricher implements RangerContextEnri return ret; } + public long getLongOption(String name, long defaultValue) { + long ret = defaultValue; + String val = getOption(name); + + if(val != null) { + ret = Long.parseLong(val); + } + + return ret; + } + public Properties readProperties(String fileName) { Properties ret = null; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java index a614a0f..7653789 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerContextEnricher.java @@ -31,9 +31,9 @@ public interface RangerContextEnricher { void setContextServiceDef(RangerServiceDef serviceDef); - void setContextComponentServiceName(String componentServiceName); + //void setContextComponentServiceName(String componentServiceName); - void setContextComponentServiceDef(RangerServiceDef componentServiceDef); + //void setContextComponentServiceDef(RangerServiceDef componentServiceDef); void init(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java new file mode 100644 index 0000000..c5a5828 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.contextenricher; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.model.RangerResource; +import org.apache.ranger.plugin.store.file.TagFileStore; + +import java.util.List; +import java.util.Map; + +public class RangerTagFileStoreRetriever extends RangerTagRefresher { + private static final Log LOG = LogFactory.getLog(RangerTagFileStoreRetriever.class); + + private final String componentType; + private final String tagServiceName; + private RangerTagReceiver receiver; + private TagFileStore tagFileStore; + + public RangerTagFileStoreRetriever(final String componentType, final String tagServiceName, final long pollingIntervalMs, final RangerTagReceiver enricher) { + super(pollingIntervalMs); + this.componentType = componentType; + this.tagServiceName = tagServiceName; + setReceiver(enricher); + } + + @Override + public void init(Map<String, Object> options) { + tagFileStore = TagFileStore.getInstance(); + } + + @Override + public void setReceiver(RangerTagReceiver receiver) { + this.receiver = receiver; + } + + @Override + public void retrieveTags() { + if (tagFileStore != null) { + List<RangerResource> resources = null; + + try { + resources = tagFileStore.getResources(tagServiceName, componentType); + } catch (Exception exp) { + LOG.error("RangerTagFileStoreRetriever.retrieveTags() - Error retrieving resources"); + } + + if (receiver != null) { + receiver.setRangerResources(resources); + } else { + LOG.error("RangerTagFileStoreRetriever.retrieveTags() - No receiver to send resources to !!"); + } + } else { + LOG.error("RangerTagFileStoreRetriever.retrieveTags() - No TagFileStore ..."); + } + } + +} + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java new file mode 100644 index 0000000..9d2b60b --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.contextenricher; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.model.RangerResource; +import org.apache.ranger.plugin.policyengine.RangerAccessRequest; +import org.apache.ranger.plugin.policyengine.RangerAccessResource; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; +import org.apache.ranger.plugin.policyresourcematcher.RangerDefaultPolicyResourceMatcher; +import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; + +import java.util.ArrayList; +import java.util.List; + +public class RangerTagProvider extends RangerAbstractContextEnricher implements RangerTagReceiver { + private static final Log LOG = LogFactory.getLog(RangerTagProvider.class); + + public enum TagProviderTypeEnum { + INVALID_TAG_PROVIDER, + FILESTORE_BASED_TAG_PROVIDER, + RANGER_ADMIN_TAG_PROVIDER, + EXTERNAL_SYSTEM_TAG_PROVIDER + } + + protected TagProviderTypeEnum tagProviderType = TagProviderTypeEnum.INVALID_TAG_PROVIDER; + protected RangerTagRefresher tagRefresher; + List<RangerTaggedResourceMatcher> taggedResourceMatchers; + + @Override + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagProvider.init()"); + } + + super.init(); + + String tagProviderTypeString = getOption("TagProviderType", "FILE_BASED_TAG_PROVIDER"); + long pollingIntervalMs = getLongOption("pollingInterval", 60 * 1000); + + if (tagProviderTypeString.equals(TagProviderTypeEnum.FILESTORE_BASED_TAG_PROVIDER.toString())) { + tagRefresher = new RangerTagFileStoreRetriever(serviceDef.getName(), serviceName, pollingIntervalMs, this); + tagProviderType = TagProviderTypeEnum.FILESTORE_BASED_TAG_PROVIDER; + } else if (tagProviderTypeString.equals(TagProviderTypeEnum.RANGER_ADMIN_TAG_PROVIDER.toString())) { + // TODO + tagProviderType = TagProviderTypeEnum.RANGER_ADMIN_TAG_PROVIDER; + } else if (tagProviderTypeString.equals(TagProviderTypeEnum.EXTERNAL_SYSTEM_TAG_PROVIDER.toString())) { + // TODO + tagProviderType = TagProviderTypeEnum.EXTERNAL_SYSTEM_TAG_PROVIDER; + } else { + LOG.error("RangerTagProvider.init() - Invalid Tag Provider.. "); + } + + // Provide additional options + if (tagRefresher != null) { + tagRefresher.init(null); + tagRefresher.retrieveTags(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagProvider.init() - Tag Provider Type:" + tagProviderType); + } + } + + @Override + public void enrich(RangerAccessRequest request) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagProvider.enrich(" + request + ")"); + } + + List<RangerTaggedResourceMatcher> taggedResourceMatchersCopy = taggedResourceMatchers; + + List<RangerResource.RangerResourceTag> matchedTags = findMatchingTags(request.getResource(), taggedResourceMatchersCopy); + + if (CollectionUtils.isNotEmpty(matchedTags)) { + request.getContext().put(RangerPolicyEngine.KEY_CONTEXT_TAGS, matchedTags); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagProvider.enrich(" + request + ")"); + } + } + + @Override + public void setRangerResources(final List<RangerResource> resources) { + + List<RangerTaggedResourceMatcher> resourceMatchers = new ArrayList<RangerTaggedResourceMatcher>(); + + if (CollectionUtils.isNotEmpty(resources)) { + + for (RangerResource taggedResource : resources) { + RangerDefaultPolicyResourceMatcher matcher = new RangerDefaultPolicyResourceMatcher(); + matcher.setServiceDef(this.serviceDef); + + + matcher.setPolicyResources(taggedResource.getResourceSpec()); + matcher.init(); + + RangerTaggedResourceMatcher taggedResourceMatcher = new RangerTaggedResourceMatcher(taggedResource, matcher); + resourceMatchers.add(taggedResourceMatcher); + + } + } + + if (CollectionUtils.isNotEmpty(resourceMatchers)) { + taggedResourceMatchers = resourceMatchers; + } + + if (tagRefresher != null && !tagRefresher.getIsStarted()) { + tagRefresher.startRetriever(); + } + } + + static private List<RangerResource.RangerResourceTag> findMatchingTags(final RangerAccessResource resource, final List<RangerTaggedResourceMatcher> resourceMatchers) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagProvider.findMatchingTags(" + resource + ")"); + } + + List<RangerResource.RangerResourceTag> ret = null; + + if (CollectionUtils.isNotEmpty(resourceMatchers)) { + + for (RangerTaggedResourceMatcher resourceMatcher : resourceMatchers) { + + RangerResource taggedResource = resourceMatcher.getRangerResource(); + RangerPolicyResourceMatcher matcher = resourceMatcher.getPolicyResourceMatcher(); + + boolean isMatched = matcher.isMatch(resource); + + if (isMatched) { + if (ret == null) { + ret = new ArrayList<RangerResource.RangerResourceTag>(); + } + ret.addAll(taggedResource.getTags()); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagProvider.findMatchingTags(" + resource + ")"); + } + + return ret; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagReceiver.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagReceiver.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagReceiver.java new file mode 100644 index 0000000..6a3bb07 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagReceiver.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.contextenricher; + +import org.apache.ranger.plugin.model.RangerResource; + +import java.util.List; + +public interface RangerTagReceiver { + void setRangerResources(final List<RangerResource> resources); +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRefresher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRefresher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRefresher.java new file mode 100644 index 0000000..f174cc8 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRefresher.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.contextenricher; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public abstract class RangerTagRefresher extends Thread implements RangerTagRetriever { + private static final Log LOG = LogFactory.getLog(RangerTagRefresher.class); + + private long pollingIntervalMs; + private volatile boolean isStarted = false; + + protected RangerTagRefresher(final long pollingIntervalMs) { + this.pollingIntervalMs = pollingIntervalMs; + } + + public final long getPollingIntervalMs() { + return pollingIntervalMs; + } + + public final void setPollingIntervalMs(long pollingIntervalMs) { + this.pollingIntervalMs = pollingIntervalMs; + } + + public boolean getIsStarted() { return isStarted; } + + public final void cleanup() { this.stopRetriever(); } + + @Override + public void run() { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerTagRefresher(pollingIntervalMs=" + pollingIntervalMs + ").run()"); + } + + while (true) { + + retrieveTags(); + + if (pollingIntervalMs > 0) { + try { + Thread.sleep(pollingIntervalMs); + } catch (InterruptedException excp) { + LOG.info("RangerTagRefresher(pollingIntervalMs=" + pollingIntervalMs + ").run() : interrupted! Exiting thread", excp); + break; + } + } else { + break; + } + + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerTagRefresher().run()"); + } + } + + public final boolean startRetriever() { + + boolean ret = isStarted; + + if (!ret) { + synchronized (this) { + + ret = isStarted; + + if (!ret) { + try { + super.start(); + ret = isStarted = true; + } catch (Exception excp) { + LOG.error("RangerTagRefresher.startRetriever() - failed to start, exception=" + excp); + } + } + } + } + + return ret; + } + + public final void stopRetriever() { + + super.interrupt(); + + try { + super.join(); + } catch (InterruptedException excp) { + LOG.error("RangerTagRefresher(): error while waiting for thread to exit", excp); + } + } +} + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java new file mode 100644 index 0000000..9176664 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.contextenricher; + +import java.util.Map; + +public interface RangerTagRetriever { + + void init (Map<String, Object> options); + + void setReceiver(RangerTagReceiver receiver); + + void retrieveTags(); +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTaggedResourceMatcher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTaggedResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTaggedResourceMatcher.java new file mode 100644 index 0000000..de41106 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTaggedResourceMatcher.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.contextenricher; + +import org.apache.ranger.plugin.model.RangerResource; +import org.apache.ranger.plugin.policyresourcematcher.RangerPolicyResourceMatcher; + +public class RangerTaggedResourceMatcher { + private final RangerResource rangerResource; + private final RangerPolicyResourceMatcher policyResourceMatcher; + + public RangerTaggedResourceMatcher(final RangerResource rangerResource, RangerPolicyResourceMatcher policyResourceMatcher) { + this.rangerResource = rangerResource; + this.policyResourceMatcher = policyResourceMatcher; + } + + public RangerResource getRangerResource() { return rangerResource; } + + public RangerPolicyResourceMatcher getPolicyResourceMatcher() { return policyResourceMatcher; } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResourceReadOnly.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResourceReadOnly.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResourceReadOnly.java index d304a47..59e968d 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResourceReadOnly.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResourceReadOnly.java @@ -19,11 +19,11 @@ package org.apache.ranger.plugin.policyengine; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.ranger.plugin.model.RangerServiceDef; -import java.util.Collections; -import java.util.Map; -import java.util.Set; +import java.util.*; public class RangerAccessResourceReadOnly implements RangerAccessResource { @@ -35,8 +35,19 @@ public class RangerAccessResourceReadOnly implements RangerAccessResource { this.source = source; // Cached here for reducing access overhead - this.keys = Collections.unmodifiableSet(source.getKeys()); - this.map = Collections.unmodifiableMap(source.getAsMap()); + Set<String> sourceKeys = source.getKeys(); + + if (CollectionUtils.isEmpty(sourceKeys)) { + sourceKeys = new HashSet<String>(); + } + this.keys = Collections.unmodifiableSet(sourceKeys); + + Map<String, String> sourceMap = source.getAsMap(); + + if (MapUtils.isEmpty(sourceMap)) { + sourceMap = new HashMap<String, String>(); + } + this.map = Collections.unmodifiableMap(sourceMap); } public String getOwnerUser() { return source.getOwnerUser(); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index 9fc4ab6..63a63d1 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@ -33,7 +33,6 @@ import org.apache.ranger.plugin.util.ServicePolicies; import java.util.*; - public class RangerPolicyEngineImpl implements RangerPolicyEngine { private static final Log LOG = LogFactory.getLog(RangerPolicyEngineImpl.class); @@ -42,10 +41,6 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { private final List<RangerContextEnricher> allContextEnrichers; - public RangerPolicyEngineImpl(ServicePolicies servicePolicies) { - this(servicePolicies, null); - } - public RangerPolicyEngineImpl(ServicePolicies servicePolicies, RangerPolicyEngineOptions options) { if (LOG.isDebugEnabled()) { LOG.debug("==> RangerPolicyEngineImpl(" + servicePolicies + ", " + options + ")"); @@ -68,8 +63,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { if (LOG.isDebugEnabled()) { LOG.debug("RangerPolicyEngineImpl : Building tag-policy-repository for tag-service " + tagPolicies.getServiceName()); } - tagPolicyRepository = new RangerPolicyRepository(tagPolicies, options, servicePolicies.getServiceName(), - servicePolicies.getServiceDef()); + + tagPolicyRepository = new RangerPolicyRepository(tagPolicies, options, servicePolicies.getServiceDef()); } else { if (LOG.isDebugEnabled()) { @@ -318,7 +313,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { if (ret != null && request != null) { - if (tagPolicyRepository != null) { + if (tagPolicyRepository != null && CollectionUtils.isNotEmpty(tagPolicyRepository.getPolicies())) { RangerAccessResult tagAccessResult = isAccessAllowedForTagPolicies(request); @@ -394,7 +389,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { LOG.debug("RangerPolicyEngineImpl.isAccessAllowedForTagPolicies: Evaluating policies for tag (" + resourceTag.getName() + ")"); } - RangerAccessRequest tagEvalRequest = new RangerTagAccessRequest(resourceTag, getServiceDef().getName(), request); + RangerAccessRequest tagEvalRequest = new RangerTagAccessRequest(resourceTag, tagPolicyRepository.getServiceDef(), request); + RangerAccessResult tagEvalResult = createAccessResult(tagEvalRequest); for (RangerPolicyEvaluator evaluator : evaluators) { @@ -501,18 +497,19 @@ class RangerTagResource extends RangerAccessResourceImpl { private static final String KEY_TAG = "tag"; - public RangerTagResource(String tagName) { + public RangerTagResource(String tagName, RangerServiceDef tagServiceDef) { super.setValue(KEY_TAG, tagName); + super.setServiceDef(tagServiceDef); } } class RangerTagAccessRequest extends RangerAccessRequestImpl { - public RangerTagAccessRequest(RangerResource.RangerResourceTag resourceTag, String componentName, RangerAccessRequest request) { - super.setResource(new RangerTagResource(resourceTag.getName())); + public RangerTagAccessRequest(RangerResource.RangerResourceTag resourceTag, RangerServiceDef tagServiceDef, RangerAccessRequest request) { + super.setResource(new RangerTagResource(resourceTag.getName(), tagServiceDef)); super.setUser(request.getUser()); super.setUserGroups(request.getUserGroups()); super.setAction(request.getAction()); - super.setAccessType(componentName + ":" + request.getAccessType()); + super.setAccessType(request.getAccessType()); super.setAccessTime(request.getAccessTime()); super.setRequestData(request.getRequestData()); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index 115d05e..ed315fd 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@ -30,12 +30,10 @@ import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.policyevaluator.RangerCachedPolicyEvaluator; import org.apache.ranger.plugin.policyevaluator.RangerOptimizedPolicyEvaluator; import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator; +import org.apache.ranger.plugin.store.AbstractServiceStore; import org.apache.ranger.plugin.util.ServicePolicies; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; public class RangerPolicyRepository { private static final Log LOG = LogFactory.getLog(RangerPolicyRepository.class); @@ -44,17 +42,16 @@ public class RangerPolicyRepository { private final RangerServiceDef serviceDef; private final List<RangerPolicy> policies; private final long policyVersion; - private List<RangerContextEnricher> contextEnrichers; - private List<RangerPolicyEvaluator> policyEvaluators; + private List<RangerContextEnricher> contextEnrichers; + private List<RangerPolicyEvaluator> policyEvaluators; private final Map<String, Boolean> accessAuditCache; - private final String componentServiceName; private final RangerServiceDef componentServiceDef; RangerPolicyRepository(ServicePolicies servicePolicies, RangerPolicyEngineOptions options) { super(); - this.componentServiceName = this.serviceName = servicePolicies.getServiceName(); + this.serviceName = servicePolicies.getServiceName(); this.componentServiceDef = this.serviceDef = servicePolicies.getServiceDef(); this.policies = Collections.unmodifiableList(servicePolicies.getPolicies()); this.policyVersion = servicePolicies.getPolicyVersion() != null ? servicePolicies.getPolicyVersion() : -1; @@ -78,14 +75,14 @@ public class RangerPolicyRepository { } - RangerPolicyRepository(ServicePolicies.TagPolicies tagPolicies, RangerPolicyEngineOptions options, String componentServiceName, + RangerPolicyRepository(ServicePolicies.TagPolicies tagPolicies, RangerPolicyEngineOptions options, RangerServiceDef componentServiceDef) { super(); this.serviceName = tagPolicies.getServiceName(); - this.serviceDef = tagPolicies.getServiceDef(); - this.componentServiceName = componentServiceName; + + this.serviceDef = normalizeAccessTypeDefs(tagPolicies.getServiceDef(), componentServiceDef.getName()); this.componentServiceDef = componentServiceDef; - this.policies = Collections.unmodifiableList(tagPolicies.getPolicies()); + this.policies = Collections.unmodifiableList(normalizePolicyItemAccesses(tagPolicies.getPolicies(), componentServiceDef.getName())); this.policyVersion = tagPolicies.getPolicyVersion() != null ? tagPolicies.getPolicyVersion() : -1; this.accessAuditCache = null; @@ -117,27 +114,120 @@ public class RangerPolicyRepository { return policyEvaluators; } - public RangerServiceDef getComponentServiceDef() { return this.componentServiceDef; } + private RangerServiceDef normalizeAccessTypeDefs(RangerServiceDef serviceDef, final String componentType) { - public String getComponentServiceName() { return this.componentServiceName; } + if (serviceDef != null && StringUtils.isNotBlank(componentType)) { - private void init(RangerPolicyEngineOptions options) { + List<RangerServiceDef.RangerAccessTypeDef> accessTypeDefs = serviceDef.getAccessTypes(); - List<RangerContextEnricher> contextEnrichers = new ArrayList<RangerContextEnricher>(); - if (!options.disableContextEnrichers && !CollectionUtils.isEmpty(serviceDef.getContextEnrichers())) { - for (RangerServiceDef.RangerContextEnricherDef enricherDef : serviceDef.getContextEnrichers()) { - if (enricherDef == null) { - continue; + if (CollectionUtils.isNotEmpty(accessTypeDefs)) { + + String prefix = componentType + AbstractServiceStore.COMPONENT_ACCESSTYPE_SEPARATOR; + + List<RangerServiceDef.RangerAccessTypeDef> unneededAccessTypeDefs = new ArrayList<RangerServiceDef.RangerAccessTypeDef>(); + + for (RangerServiceDef.RangerAccessTypeDef accessTypeDef : accessTypeDefs) { + + String accessType = accessTypeDef.getName(); + + if (StringUtils.startsWith(accessType, prefix)) { + + String newAccessType = StringUtils.removeStart(accessType, prefix); + + accessTypeDef.setName(newAccessType); + + Collection<String> impliedGrants = accessTypeDef.getImpliedGrants(); + + if (CollectionUtils.isNotEmpty(impliedGrants)) { + + Collection<String> newImpliedGrants = null; + + for (String impliedGrant : impliedGrants) { + + if (StringUtils.startsWith(impliedGrant, prefix)) { + + String newImpliedGrant = StringUtils.removeStart(impliedGrant, prefix); + + if (newImpliedGrants == null) { + newImpliedGrants = new ArrayList<String>(); + } + + newImpliedGrants.add(newImpliedGrant); + } + } + accessTypeDef.setImpliedGrants(newImpliedGrants); + + } + } else if (StringUtils.contains(accessType, AbstractServiceStore.COMPONENT_ACCESSTYPE_SEPARATOR)) { + unneededAccessTypeDefs.add(accessTypeDef); + } } + accessTypeDefs.removeAll(unneededAccessTypeDefs); + } + } + + return serviceDef; + } + + private List<RangerPolicy> normalizePolicyItemAccesses(List<RangerPolicy> rangerPolicies, final String componentType) { + + if (CollectionUtils.isNotEmpty(rangerPolicies) && StringUtils.isNotBlank(componentType)) { + + String prefix = componentType + AbstractServiceStore.COMPONENT_ACCESSTYPE_SEPARATOR; + + List<RangerPolicy> unneededPolicies = new ArrayList<>(); - RangerContextEnricher contextEnricher = buildContextEnricher(enricherDef); + for (RangerPolicy policy : rangerPolicies) { - if(contextEnricher != null) { - contextEnrichers.add(contextEnricher); + List<RangerPolicy.RangerPolicyItem> policyItems = policy.getPolicyItems(); + + if (CollectionUtils.isNotEmpty(policyItems)) { + + List<RangerPolicy.RangerPolicyItem> unneededPolicyItems = new ArrayList< RangerPolicy.RangerPolicyItem>(); + + for (RangerPolicy.RangerPolicyItem policyItem : policyItems) { + + List<RangerPolicy.RangerPolicyItemAccess> policyItemAccesses = policyItem.getAccesses(); + + if (CollectionUtils.isNotEmpty(policyItemAccesses)) { + + List<RangerPolicy.RangerPolicyItemAccess> unneededItemAccesses = new ArrayList<RangerPolicy.RangerPolicyItemAccess>(); + + for (RangerPolicy.RangerPolicyItemAccess access : policyItemAccesses) { + + String accessType = access.getType(); + + if (StringUtils.startsWith(accessType, prefix)) { + + String newAccessType = StringUtils.removeStart(accessType, prefix); + + access.setType(newAccessType); + + } else if (accessType.contains(AbstractServiceStore.COMPONENT_ACCESSTYPE_SEPARATOR)) { + unneededItemAccesses.add(access); + } + } + policyItemAccesses.removeAll(unneededItemAccesses); + + if (policyItemAccesses.isEmpty() && !policyItem.getDelegateAdmin()) { + unneededPolicyItems.add(policyItem); + } + } + } + + policyItems.removeAll(unneededPolicyItems); + } + if (CollectionUtils.isEmpty(policyItems) && !policy.getIsAuditEnabled()) { + unneededPolicies.add(policy); } } + rangerPolicies.removeAll(unneededPolicies); } - this.contextEnrichers = Collections.unmodifiableList(contextEnrichers); + + return rangerPolicies; + } + + private void init(RangerPolicyEngineOptions options) { List<RangerPolicyEvaluator> policyEvaluators = new ArrayList<RangerPolicyEvaluator>(); for (RangerPolicy policy : policies) { @@ -154,6 +244,24 @@ public class RangerPolicyRepository { Collections.sort(policyEvaluators); this.policyEvaluators = Collections.unmodifiableList(policyEvaluators); + List<RangerContextEnricher> contextEnrichers = new ArrayList<RangerContextEnricher>(); + if (CollectionUtils.isNotEmpty(this.policyEvaluators)) { + if (!options.disableContextEnrichers && !CollectionUtils.isEmpty(serviceDef.getContextEnrichers())) { + for (RangerServiceDef.RangerContextEnricherDef enricherDef : serviceDef.getContextEnrichers()) { + if (enricherDef == null) { + continue; + } + + RangerContextEnricher contextEnricher = buildContextEnricher(enricherDef); + + if (contextEnricher != null) { + contextEnrichers.add(contextEnricher); + } + } + } + } + this.contextEnrichers = Collections.unmodifiableList(contextEnrichers); + } private RangerContextEnricher buildContextEnricher(RangerServiceDef.RangerContextEnricherDef enricherDef) { @@ -180,9 +288,7 @@ public class RangerPolicyRepository { if(ret != null) { ret.setContextEnricherDef(enricherDef); ret.setContextServiceName(serviceName); - ret.setContextServiceDef(serviceDef); - ret.setContextComponentServiceName(componentServiceName); - ret.setContextComponentServiceDef(componentServiceDef); + ret.setContextServiceDef(componentServiceDef); ret.init(); } @@ -273,9 +379,6 @@ public class RangerPolicyRepository { sb.append("serviceName={").append(serviceName).append("} "); sb.append("serviceDef={").append(serviceDef).append("} "); - sb.append("componentServiceName={").append(componentServiceName).append("} "); - sb.append("componentServiceDef={").append(componentServiceDef).append("} "); - sb.append("policyEvaluators={"); if (policyEvaluators != null) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java index 86dfd1f..837fce8 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java @@ -40,6 +40,7 @@ import java.util.UUID; public abstract class AbstractServiceStore implements ServiceStore { private static final Log LOG = LogFactory.getLog(AbstractServiceStore.class); + public static final String COMPONENT_ACCESSTYPE_SEPARATOR = ":"; private static final int MAX_ACCESS_TYPES_IN_SERVICE_DEF = 1000; @@ -314,7 +315,7 @@ public abstract class AbstractServiceStore implements ServiceStore { tagAccessType.setImpliedGrants(new HashSet<String>()); if(CollectionUtils.isNotEmpty(svcAccessType.getImpliedGrants())) { for(String svcImpliedGrant : svcAccessType.getImpliedGrants()) { - tagAccessType.getImpliedGrants().add(serviceDefName + ":" + svcImpliedGrant); + tagAccessType.getImpliedGrants().add(serviceDefName + COMPONENT_ACCESSTYPE_SEPARATOR + svcImpliedGrant); } } @@ -323,7 +324,7 @@ public abstract class AbstractServiceStore implements ServiceStore { } for (RangerServiceDef.RangerAccessTypeDef tagAccessType : tagDefAccessTypes) { - if (tagAccessType.getName().startsWith(serviceDefName + ":")) { + if (tagAccessType.getName().startsWith(serviceDefName + COMPONENT_ACCESSTYPE_SEPARATOR)) { long svcAccessTypeItemId = tagAccessType.getItemId() - itemIdOffset; RangerServiceDef.RangerAccessTypeDef svcAccessType = findAccessTypeDef(svcAccessTypeItemId, svcDefAccessTypes); @@ -352,7 +353,7 @@ public abstract class AbstractServiceStore implements ServiceStore { isUpdated = true; } else if(tagImpliedGrantsLen > 0) { for(String svcImpliedGrant : svcImpliedGrants) { - if(! tagImpliedGrants.contains(serviceDefName + ":" + svcImpliedGrant)) { + if(! tagImpliedGrants.contains(serviceDefName + COMPONENT_ACCESSTYPE_SEPARATOR + svcImpliedGrant)) { isUpdated = true; break; } @@ -361,14 +362,14 @@ public abstract class AbstractServiceStore implements ServiceStore { } if(isUpdated) { - tagAccessType.setName(serviceDefName + ":" + svcAccessType.getName()); + tagAccessType.setName(serviceDefName + COMPONENT_ACCESSTYPE_SEPARATOR + svcAccessType.getName()); tagAccessType.setLabel(svcAccessType.getLabel()); tagAccessType.setRbKeyLabel(svcAccessType.getRbKeyLabel()); tagAccessType.setImpliedGrants(new HashSet<String>()); if(CollectionUtils.isNotEmpty(svcAccessType.getImpliedGrants())) { for(String svcImpliedGrant : svcAccessType.getImpliedGrants()) { - tagAccessType.getImpliedGrants().add(serviceDefName + ":" + svcImpliedGrant); + tagAccessType.getImpliedGrants().add(serviceDefName + COMPONENT_ACCESSTYPE_SEPARATOR + svcImpliedGrant); } } @@ -413,7 +414,7 @@ public abstract class AbstractServiceStore implements ServiceStore { List<RangerServiceDef.RangerAccessTypeDef> accessTypes = new ArrayList<RangerServiceDef.RangerAccessTypeDef>(); for (RangerServiceDef.RangerAccessTypeDef accessType : tagServiceDef.getAccessTypes()) { - if (accessType.getName().startsWith(serviceDefName + ":")) { + if (accessType.getName().startsWith(serviceDefName + COMPONENT_ACCESSTYPE_SEPARATOR)) { accessTypes.add(accessType); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java index 558014b..d23744b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java @@ -96,13 +96,6 @@ public abstract class AbstractTagStore implements TagStore { (long)list.size(), list.size(), filter.getSortType(), filter.getSortBy()); } - public PList<RangerResource> getPaginatedResources(String tagServiceName, String serviceType) throws Exception { - List<RangerResource> list = getResources(tagServiceName, serviceType); - - return new PList<RangerResource>(list, 0, list.size(), - (long)list.size(), list.size(), null, null); - } - public PList<RangerResource> getPaginatedResources(SearchFilter filter) throws Exception { List<RangerResource> list = getResources(filter); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java index fd48d63..2c9ccc0 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java @@ -46,7 +46,7 @@ public class TagPredicateUtil extends AbstractPredicateUtil { addPredicateForTagDefName(filter.getParam(SearchFilter.TAG_DEF_NAME), predicates); addPredicateForTagResourceServiceName(filter.getParam(SearchFilter.TAG_RESOURCE_SERVICE_NAME), predicates); - addPredicateForTagResourceServiceType(filter.getParam(SearchFilter.TAG_RESOURCE_SERVICE_TYPE), predicates); + addPredicateForTagResourceComponentType(filter.getParam(SearchFilter.TAG_RESOURCE_COMPONENT_TYPE), predicates); addPredicateForTagResourceId(filter.getParam(SearchFilter.TAG_RESOURCE_ID), predicates); } @@ -147,7 +147,7 @@ public class TagPredicateUtil extends AbstractPredicateUtil { return ret; } - private Predicate addPredicateForTagResourceServiceType(final String type, List<Predicate> predicates) { + private Predicate addPredicateForTagResourceComponentType(final String type, List<Predicate> predicates) { if (type == null || StringUtils.isEmpty(type)) { return null; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java index 991e913..a43cbf9 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java @@ -54,11 +54,10 @@ public interface TagStore { RangerResource getResource(Long id) throws Exception; - List<RangerResource> getResources(String tagServiceName, String serviceType) throws Exception; - - PList<RangerResource> getPaginatedResources(String tagServiceName, String serviceType) throws Exception; + List<RangerResource> getResources(String tagServiceName, String componentType) throws Exception; List<RangerResource> getResources(SearchFilter filter) throws Exception; + PList<RangerResource> getPaginatedResources(SearchFilter filter) throws Exception; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java index 48059ce..ad0bc66 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java @@ -408,9 +408,9 @@ public class TagFileStore extends AbstractTagStore { } @Override - public List<RangerResource> getResources(String tagServiceName, String serviceType) throws Exception { + public List<RangerResource> getResources(String tagServiceName, String componentType) throws Exception { if (LOG.isDebugEnabled()) { - LOG.debug("==> TagFileStore.getResources(" + tagServiceName + ", " + serviceType + ")"); + LOG.debug("==> TagFileStore.getResources(" + tagServiceName + ", " + componentType + ")"); } List<RangerResource> ret; @@ -420,14 +420,14 @@ public class TagFileStore extends AbstractTagStore { filter.setParam(SearchFilter.TAG_RESOURCE_SERVICE_NAME, tagServiceName); } - if (StringUtils.isNotBlank(serviceType)) { - filter.setParam(SearchFilter.TAG_RESOURCE_SERVICE_TYPE, serviceType); + if (StringUtils.isNotBlank(componentType)) { + filter.setParam(SearchFilter.TAG_RESOURCE_COMPONENT_TYPE, componentType); } ret = getResources(filter); if (LOG.isDebugEnabled()) { - LOG.debug("<== TagFileStore.getResources(" + tagServiceName + ", " + serviceType + ")"); + LOG.debug("<== TagFileStore.getResources(" + tagServiceName + ", " + componentType + ")"); } return ret; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java index b987904..1aadbf5 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java @@ -54,7 +54,7 @@ public class SearchFilter { public static final String TAG_DEF_NAME = "tagDefName"; // search public static final String TAG_RESOURCE_ID = "tagResourceId"; // search public static final String TAG_RESOURCE_SERVICE_NAME = "tagResourceServiceName"; // search - public static final String TAG_RESOURCE_SERVICE_TYPE = "tagResourceServiceType"; // search + public static final String TAG_RESOURCE_COMPONENT_TYPE = "tagResourceComponentType"; // search http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/main/resources/service-defs/ranger-servicedef-_tag_.json ---------------------------------------------------------------------- diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-_tag_.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-_tag_.json index 0fa5577..10a4ee5 100644 --- a/agents-common/src/main/resources/service-defs/ranger-servicedef-_tag_.json +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-_tag_.json @@ -47,11 +47,26 @@ "contextEnrichers": [ - + { + "itemId":1, + "name" : "TagEnricher", + "enricher" : "org.apache.ranger.plugin.contextenricher.RangerTagProvider", + "enricherOptions" : { + "TagProviderType":"FILESTORE_BASED_TAG_PROVIDER", + "pollingInterval":10000 + } + } ], "policyConditions": [ - + { + "itemId":1, + "name":"ScriptConditionEvaluator", + "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerScriptConditionEvaluator", + "evaluatorOptions" : {"engineName":"JavaScript"}, + "label":"Script", + "description": "Script to execute" + } ] } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json ---------------------------------------------------------------------- diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json b/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json index 46f95a4..46a9eb1 100644 --- a/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json +++ b/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json @@ -67,23 +67,45 @@ ], "accessTypes": [ { + "itemId": 1, "name": "hdfs:read", "label": "hdfs:Read" }, { + "itemId": 2, "name": "hdfs:write", "label": "hdfs:Write" }, { + "itemId": 3, "name": "hdfs:execute", "label": "hdfs:Execute" } + , + { + "itemId": 4, + "name": "hive:grant", + "label": "hive:grant" + } + , + { + "itemId": 5, + "name": "kms:dek", + "label": "kms:dek" + } + , + { + "itemId": 6, + "name": "delete", + "label": "delete" + } ], - "contextEnrichers" : [ + "contextEnrichers": [ { - "name" : "test_enricher", - "enricher" : "org.apache.ranger.plugin.contextenricher.RangerFileBasedTagProvider", - "enricherOptions" : {"wildCard":true, "ignoreCase":true, "dataFile":"/etc/ranger/data/resourceTags.txt"} + "itemId": 1, + "name" : "TagEnricher", + "enricher" : "org.apache.ranger.plugin.contextenricher.RangerTagProvider", + "enricherOptions" : {"TagProviderType":"FILESTORE_BASED_TAG_PROVIDER", "pollingInterval":30000, "dataFile":"/etc/ranger/data/resourceTags.txt"} } ], "policyConditions": [ @@ -101,16 +123,18 @@ {"id":5,"name":"allow somethingelse tag","isEnabled":true,"isAuditEnabled":true,"policyType":0, "resources":{"tag":{"values":["restricte?"],"isRecursive":false}}, "policyItems":[ - {"accesses":[{"type":"hdfs:read","isAllowed":true}],"users":["user1"],"groups":["finance"],"delegateAdmin":false, + {"accesses":[{"type":"hdfs:read", "isAllowed":true}, {"type":"hive:grant", "isAllowed":true}, {"type":"delete", "isAllowed":true}, {"type":":write", "isAllowed":true}],"users":["user1"],"groups":["finance"],"delegateAdmin":false, "conditions" : [{"type":"ScriptConditionEvaluator", "values": [ - "var accessTime = request.getAccessTime(); println('accessTime=' + accessTime); result = true;" + "importPackage(java.util); var accessTime = ctx.accessTime; print('accessTime=' + accessTime); var accessDate = ctx.getAsDate(accessTime); println('accessDate=' + accessDate); var currentDate = new Date(); println('currentDate=' + currentDate); println('current=' +currentDate.getTime()); println('access='+ accessDate.getTime()); result = true;" , - "var accessTime = request.accessTime; println('accessTime=' + accessTime); result = true;" + "importPackage(java.util); var accessTime = ctx.accessTime; var currentTime = Date(); print('accessTime=' + accessTime); println('currentTime=' + currentTime); accessTime.after(currentTime)" , "var ownerUser = request.resource.getOwnerUser(); println('ownerUser=' + ownerUser); result = true;" , "var ownerUser = request.resource.ownerUser; println('ownerUser=' + ownerUser); result = true;" , + "var tagName = ctx.currentTagName; println('tagName=' + tagName); ctx.result = true;" + , "var resource = ctx.get('RESOURCE').getAsMap(); println('resource path=' + resource.get('path')); result = true;" ] http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java b/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java index d093a35..c8e387c 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java @@ -375,17 +375,17 @@ public class TagREST { @Path(TagRESTConstants.RESOURCES_RESOURCE) @Produces({ "application/json", "application/xml" }) public List<RangerResource> getResources(@DefaultValue("") @QueryParam(TagRESTConstants.TAG_SERVICE_NAME_PARAM) String tagServiceName, - @DefaultValue("") @QueryParam(TagRESTConstants.SERVICE_TYPE_PARAM) String serviceType) { + @DefaultValue("") @QueryParam(TagRESTConstants.COMPONENT_TYPE_PARAM) String componentType) { if(LOG.isDebugEnabled()) { - LOG.debug("==> TagREST.getResources(" + tagServiceName + ", " + serviceType + ")"); + LOG.debug("==> TagREST.getResources(" + tagServiceName + ", " + componentType + ")"); } List<RangerResource> ret; try { - ret = tagStore.getResources(tagServiceName, serviceType); + ret = tagStore.getResources(tagServiceName, componentType); } catch(Exception excp) { - LOG.error("getResources(" + tagServiceName + ", " + serviceType + ") failed", excp); + LOG.error("getResources(" + tagServiceName + ", " + componentType + ") failed", excp); throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e25baa77/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java b/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java index 95d1c1e..27c9b19 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java @@ -24,6 +24,7 @@ public class TagRESTConstants { static final String TAGS_RESOURCE = "tags"; static final String TAG_RESOURCE = "tag"; static final String RESOURCES_RESOURCE = "resources"; + static final String RESOURCES_IF_UPDATED_RESOURCE = "resources"; static final String RESOURCE_RESOURCE = "resource"; static final String ACTION_SUB_RESOURCE = "update"; static final String ACTION_OP = "op"; @@ -32,7 +33,7 @@ public class TagRESTConstants { static final String ACTION_DELETE = "delete"; public static final String TAG_SERVICE_NAME_PARAM = "tagservicename"; - public static final String SERVICE_TYPE_PARAM = "servicetype"; + public static final String COMPONENT_TYPE_PARAM = "componenttype";
