Merge branch 'master' into tag-policy
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/5d437fae Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/5d437fae Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/5d437fae Branch: refs/heads/tag-policy Commit: 5d437fae5576404ed8c8c882711188c50790c1f1 Parents: 3ffde5d f35d53b Author: Madhan Neethiraj <[email protected]> Authored: Tue Jun 2 12:42:39 2015 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Tue Jun 2 12:42:39 2015 -0700 ---------------------------------------------------------------------- .gitignore | 2 + .../audit/destination/DBAuditDestination.java | 3 +- .../audit/destination/FileAuditDestination.java | 3 +- .../audit/destination/HDFSAuditDestination.java | 7 ++- .../destination/Log4JAuditDestination.java | 15 ++++-- .../audit/destination/SolrAuditDestination.java | 8 ++- .../ranger/audit/provider/BaseAuditHandler.java | 14 ++++- .../audit/provider/MultiDestAuditProvider.java | 21 ++++++-- .../ranger/audit/queue/AuditBatchQueue.java | 5 +- .../apache/ranger/audit/queue/AuditQueue.java | 26 ++++++--- .../plugin/audit/RangerDefaultAuditHandler.java | 4 +- .../RangerFileBasedTagProvider.java | 2 +- .../policyengine/RangerAccessResource.java | 6 ++- .../policyengine/RangerAccessResourceImpl.java | 16 +++++- .../RangerAccessResourceReadOnly.java | 6 ++- .../policyengine/RangerMutableResource.java | 3 ++ .../plugin/policyengine/RangerPolicyEngine.java | 4 +- .../policyengine/RangerPolicyEngineImpl.java | 55 ++++++++++++++------ .../policyengine/RangerPolicyRepository.java | 4 +- .../RangerResourceAccessCacheImpl.java | 4 +- .../ranger/plugin/service/RangerBasePlugin.java | 14 ++--- .../plugin/policyengine/TestPolicyEngine.java | 3 +- .../hadoop/RangerHdfsAuthorizer.java | 4 +- .../hive/authorizer/RangerHiveAuditHandler.java | 4 +- .../hive/authorizer/RangerHiveAuthorizer.java | 6 +-- .../crypto/key/RangerKeyStoreProvider.java | 25 +++++---- plugin-kafka/conf/kafka-ranger-env.sh | 20 +++++++ src/main/assembly/plugin-kafka.xml | 3 -- 28 files changed, 202 insertions(+), 85 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5d437fae/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagProvider.java ---------------------------------------------------------------------- diff --cc agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagProvider.java index 5cade5b,0000000..97c82dd mode 100644,000000..100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagProvider.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagProvider.java @@@ -1,110 -1,0 +1,110 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.contextenricher; + +import java.lang.reflect.Type; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.model.RangerResource; +import org.apache.ranger.plugin.policyengine.RangerAccessRequest; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; + + +public class RangerFileBasedTagProvider extends RangerAbstractContextEnricher { + private static final Log LOG = LogFactory.getLog(RangerFileBasedTagProvider.class); + + private Properties resourceTagsMap = null; + String dataFile = null; + private Gson gsonBuilder = null; + + @Override + public void init() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerFileBasedTagProvider.init()"); + } + + super.init(); + + dataFile = getOption("dataFile", "/etc/ranger/data/resourceTags.txt"); + + resourceTagsMap = readProperties(dataFile); + + gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z") + .setPrettyPrinting() + .create(); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerFileBasedTagProvider.init()"); + } + } + + @Override + public void enrich(RangerAccessRequest request) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerFileBasedTagProvider.enrich(" + request + ")"); + } + + if(request != null && resourceTagsMap != null) { + Map<String, Object> context = request.getContext(); + /* + This needs to know about : + - componentServiceDef (to filter on component-type which is required for getting matchers), and + - serviceName (to filter on cluster-specific tags) + */ + // Provider is file-based. + // tags are a JSON strings + - String requestedResource = request.getResource().getAsString(componentServiceDef); ++ String requestedResource = request.getResource().getAsString(); + + if(LOG.isDebugEnabled()) { + LOG.debug("RangerFileBasedTagProvider.enrich(): requestedResource = '"+ requestedResource +"'"); + } + String tagsJsonString = resourceTagsMap.getProperty(requestedResource); + + if(!StringUtils.isEmpty(tagsJsonString) && context != null) { + try { + Type listType = new TypeToken<List<RangerResource.RangerResourceTag>>() { + }.getType(); + List<RangerResource.RangerResourceTag> tagList = gsonBuilder.fromJson(tagsJsonString, listType); + + context.put(RangerPolicyEngine.KEY_CONTEXT_TAGS, tagList); + } catch (Exception e) { + LOG.error("RangerFileBasedTagProvider.enrich(): error parsing file " + this.dataFile + ", exception=" + e); + } + } else { + if(LOG.isDebugEnabled()) { + LOG.debug("RangerFileBasedTagProvider.enrich(): skipping due to unavailable context or tags. context=" + context + "; tags=" + tagsJsonString); + } + } + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerFileBasedTagProvider.enrich(" + request + ")"); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5d437fae/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResource.java ---------------------------------------------------------------------- diff --cc agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResource.java index c2f4665,d645e56..8c1c7c5 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResource.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResource.java @@@ -34,13 -34,13 +34,15 @@@ public interface RangerAccessResource public abstract String getValue(String name); + public RangerServiceDef getServiceDef(); + public Set<String> getKeys(); - public String getLeafName(RangerServiceDef serviceDef); + public String getLeafName(); - public String getAsString(RangerServiceDef serviceDef); + public String getAsString(); public Map<String, String> getAsMap(); + + public RangerAccessResource getReadOnlyCopy(); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5d437fae/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResourceImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5d437fae/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResourceReadOnly.java ---------------------------------------------------------------------- diff --cc agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResourceReadOnly.java index 70e30d3,0000000..d304a47 mode 100644,000000..100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResourceReadOnly.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResourceReadOnly.java @@@ -1,57 -1,0 +1,59 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.policyengine; + +import org.apache.ranger.plugin.model.RangerServiceDef; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +public class RangerAccessResourceReadOnly implements RangerAccessResource { + + private final RangerAccessResource source; + private final Set<String> keys; + private final Map<String, String> map; + + public RangerAccessResourceReadOnly(final RangerAccessResource source) { + this.source = source; + + // Cached here for reducing access overhead + this.keys = Collections.unmodifiableSet(source.getKeys()); + this.map = Collections.unmodifiableMap(source.getAsMap()); + } + + public String getOwnerUser() { return source.getOwnerUser(); } + + public boolean exists(String name) { return source.exists(name); } + + public String getValue(String name) { return source.getValue(name); } + ++ public RangerServiceDef getServiceDef() { return source.getServiceDef(); } ++ + public Set<String> getKeys() { return keys; } + - public String getLeafName(RangerServiceDef serviceDef) { return source.getLeafName(serviceDef); } ++ public String getLeafName() { return source.getLeafName(); } + - public String getAsString(RangerServiceDef serviceDef) { return source.getAsString(serviceDef); } ++ public String getAsString() { return source.getAsString(); } + + public Map<String, String> getAsMap() { return map; } + + public RangerAccessResource getReadOnlyCopy() { return this; } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5d437fae/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5d437fae/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java ---------------------------------------------------------------------- diff --cc agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index 69757da,d2b3a5c..9fc4ab6 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@@ -120,15 -83,17 +120,17 @@@ public class RangerPolicyEngineImpl imp } @Override - public void enrichContext(RangerAccessRequest request) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> RangerPolicyEngineImpl.enrichContext(" + request + ")"); + public void preProcess(RangerAccessRequest request) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.preProcess(" + request + ")"); } + setResourceServiceDef(request); + - List<RangerContextEnricher> enrichers = policyRepository.getContextEnrichers(); + List<RangerContextEnricher> enrichers = allContextEnrichers; - if (request != null && !CollectionUtils.isEmpty(enrichers)) { - for (RangerContextEnricher enricher : enrichers) { + if(!CollectionUtils.isEmpty(enrichers)) { + for(RangerContextEnricher enricher : enrichers) { enricher.enrich(request); } } @@@ -139,17 -104,23 +141,23 @@@ } @Override - public void enrichContext(Collection<RangerAccessRequest> requests) { - if (LOG.isDebugEnabled()) { - LOG.debug("==> RangerPolicyEngineImpl.enrichContext(" + requests + ")"); + public void preProcess(Collection<RangerAccessRequest> requests) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.preProcess(" + requests + ")"); } - List<RangerContextEnricher> enrichers = allContextEnrichers; + if(CollectionUtils.isNotEmpty(requests)) { + for(RangerAccessRequest request : requests) { + setResourceServiceDef(request); + } + - List<RangerContextEnricher> enrichers = policyRepository.getContextEnrichers(); ++ List<RangerContextEnricher> enrichers = allContextEnrichers; - if (!CollectionUtils.isEmpty(requests) && !CollectionUtils.isEmpty(enrichers)) { - for (RangerContextEnricher enricher : enrichers) { - for (RangerAccessRequest request : requests) { - enricher.enrich(request); + if(CollectionUtils.isNotEmpty(enrichers)) { + for(RangerContextEnricher enricher : enrichers) { + for(RangerAccessRequest request : requests) { + enricher.enrich(request); + } } } } @@@ -353,109 -307,19 +361,122 @@@ return ret; } + protected RangerAccessResult isAccessAllowedForTagPolicies(final RangerAccessRequest request) { + + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerPolicyEngineImpl.isAccessAllowedForTagPolicies(" + request + ")"); + } + + RangerAccessResult result = createAccessResult(request); + + Map<String, Object> context = request.getContext(); + Object contextObj; + + if (context != null && (contextObj = context.get(KEY_CONTEXT_TAGS)) != null) { + + @SuppressWarnings("unchecked") + List<RangerResource.RangerResourceTag> resourceTags = (List<RangerResource.RangerResourceTag>) contextObj; + + List<RangerPolicyEvaluator> evaluators; + + if (!CollectionUtils.isEmpty(evaluators = tagPolicyRepository.getPolicyEvaluators())) { + + boolean someTagPolicyDeniedAccess = false; + boolean someTagPolicyAllowedAccess = false; + boolean someTagPolicyRequiredAudit = false; + RangerAccessResult allowedAccessResult = createAccessResult(request); + RangerAccessResult deniedAccessResult = createAccessResult(request); + + List<RangerTagAuditEvent> tagAuditEvents = new ArrayList<RangerTagAuditEvent>(); + + for (RangerResource.RangerResourceTag resourceTag : resourceTags) { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerPolicyEngineImpl.isAccessAllowedForTagPolicies: Evaluating policies for tag (" + resourceTag.getName() + ")"); + } + + RangerAccessRequest tagEvalRequest = new RangerTagAccessRequest(resourceTag, getServiceDef().getName(), request); + RangerAccessResult tagEvalResult = createAccessResult(tagEvalRequest); + + for (RangerPolicyEvaluator evaluator : evaluators) { + + evaluator.evaluate(tagEvalRequest, tagEvalResult); + + if (evaluator.isFinal() || + (tagEvalResult.getIsAccessDetermined() && tagEvalResult.getIsAuditedDetermined())) { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerPolicyEngineImpl.isAccessAllowedForTagPolicies: concluding eval for tag-policy-id=" + tagEvalResult.getPolicyId() + " for tag (" + resourceTag.getName() + ") with authorization=" + tagEvalResult.getIsAllowed()); + } + break; + } + } + + if (tagEvalResult.getIsAuditedDetermined()) { + someTagPolicyRequiredAudit = true; + // And generate an audit event + if (tagEvalResult.getIsAccessDetermined()) { + RangerTagAuditEvent event = new RangerTagAuditEvent(resourceTag.getName(), tagEvalResult); + tagAuditEvents.add(event); + } + } + + if (tagEvalResult.getIsAccessDetermined()) { + if (tagEvalResult.getIsAllowed()) { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerPolicyEngineImpl.isAccessAllowedForTagPolicies: access allowed"); + } + someTagPolicyAllowedAccess = true; + allowedAccessResult.setAccessResultFrom(tagEvalResult); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerPolicyEngineImpl.isAccessAllowedForTagPolicies: access denied"); + } + someTagPolicyDeniedAccess = true; + deniedAccessResult.setAccessResultFrom(tagEvalResult); + } + } + } + + if (someTagPolicyDeniedAccess) { + result.setAccessResultFrom(deniedAccessResult); + } else if (someTagPolicyAllowedAccess) { + result.setAccessResultFrom(allowedAccessResult); + } + + if (someTagPolicyRequiredAudit) { + if (LOG.isDebugEnabled()) { + LOG.debug("RangerPolicyEngineImpl.isAccessAllowedForTagPolicies: at least one tag-policy requires audit"); + } + result.setIsAudited(true); + RangerTagAuditEvent.processTagEvents(tagAuditEvents, someTagPolicyDeniedAccess); + // Set processed list into result + // result.setAuxilaryAuditInfo(tagAuditEvents); + } + if (LOG.isDebugEnabled()) { + LOG.debug("RangerPolicyEngineImpl.isAccessAllowedForTagPolicies() : result=" + result); + LOG.debug("RangerPolicyEngineImpl.isAccessAllowedForTagPolicies() : auditEventList=" + tagAuditEvents); + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerPolicyEngineImpl.isAccessAllowedForTagPolicies(" + request + ")" ); + } + + return result; + } + + private void setResourceServiceDef(RangerAccessRequest request) { + RangerAccessResource resource = request.getResource(); + + if (resource.getServiceDef() == null) { + if (resource instanceof RangerMutableResource) { + RangerMutableResource mutable = (RangerMutableResource) resource; + mutable.setServiceDef(getServiceDef()); + } else { + LOG.debug("RangerPolicyEngineImpl.setResourceServiceDef(): Cannot set ServiceDef in RangerResource."); + } + } + } + @Override public String toString( ) { StringBuilder sb = new StringBuilder(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5d437fae/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java ---------------------------------------------------------------------- diff --cc agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java index cc90abc,c063b94..115d05e --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java @@@ -243,8 -210,8 +243,8 @@@ public class RangerPolicyRepository LOG.debug("==> RangerPolicyRepository.storeAuditEnabledInCache()"); } - if ((ret.getIsAuditedDetermined() == true)) { + if ((ret.getIsAuditedDetermined())) { - String strResource = request.getResource().getAsString(getServiceDef()); + String strResource = request.getResource().getAsString(); Boolean value = ret.getIsAudited() ? Boolean.TRUE : Boolean.FALSE; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5d437fae/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5d437fae/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java ---------------------------------------------------------------------- diff --cc agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java index 42da42d,38b7302..f1e111e --- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java @@@ -94,27 -100,12 +94,28 @@@ public class TestPolicyEngine servicePolicies.setServiceDef(testCase.serviceDef); servicePolicies.setPolicies(testCase.policies); - policyEngine = new RangerPolicyEngineImpl(servicePolicies); + if (null != testCase.tagPolicyInfo) { + ServicePolicies.TagPolicies tagPolicies = new ServicePolicies.TagPolicies(); + tagPolicies.setServiceName(testCase.tagPolicyInfo.serviceName); + tagPolicies.setServiceDef(testCase.tagPolicyInfo.serviceDef); + tagPolicies.setPolicies(testCase.tagPolicyInfo.tagPolicies); + + servicePolicies.setTagPolicies(tagPolicies); + } + + String componentName = testCase.serviceDef.getName(); + + RangerPolicyEngineOptions policyEngineOptions = new RangerPolicyEngineOptions(); + + // Uncomment next line for testing tag-policy evaluation + // policyEngineOptions.disableTagPolicyEvaluation = false; + + policyEngine = new RangerPolicyEngineImpl(servicePolicies, policyEngineOptions); for(TestData test : testCase.tests) { + policyEngine.preProcess(test.request); + RangerAccessResult expected = test.result; - policyEngine.enrichContext(test.request); RangerAccessResult result = policyEngine.isAccessAllowed(test.request, null); assertNotNull("result was null! - " + test.name, result);
