Repository: incubator-ranger Updated Branches: refs/heads/master c38a718ed -> 2eb3c2fe6
Enhance Ranger policy engine to support X-Forwarded-For address Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/2eb3c2fe Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/2eb3c2fe Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/2eb3c2fe Branch: refs/heads/master Commit: 2eb3c2fe63c5c41a855dddd5bbb8d7a24ee226de Parents: c38a718 Author: Abhay Kulkarni <[email protected]> Authored: Wed Jun 8 17:32:48 2016 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Thu Jun 9 15:40:49 2016 -0700 ---------------------------------------------------------------------- .../ranger/audit/model/AuthzAuditEvent.java | 9 +- .../hadoop/constants/RangerHadoopConstants.java | 2 +- .../plugin/audit/RangerDefaultAuditHandler.java | 13 +++ .../policyengine/RangerAccessRequest.java | 7 +- .../policyengine/RangerAccessRequestImpl.java | 76 +++++++++++-- .../RangerAccessRequestReadOnly.java | 9 ++ .../plugin/policyengine/RangerPolicyEngine.java | 4 + .../policyengine/RangerPolicyEngineImpl.java | 18 +++ .../ranger/plugin/service/RangerBasePlugin.java | 26 ++++- .../plugin/policyengine/TestPolicyEngine.java | 113 ++++++++++++------- .../policyengine/test_policyengine_geo.json | 11 +- .../policyengine/test_policyengine_hdfs.json | 6 +- .../policyengine/test_policyengine_hive.json | 1 + 13 files changed, 238 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2eb3c2fe/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java index 55dc448..b3f80cb 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java @@ -112,6 +112,9 @@ public class AuthzAuditEvent extends AuditEventBase { @SerializedName("tags") protected Set<String> tags = new HashSet<>(); + @SerializedName("additional_info") + protected String additionalInfo; + public AuthzAuditEvent() { super(); @@ -450,6 +453,10 @@ public class AuthzAuditEvent extends AuditEventBase { this.tags = tags; } + public String getAdditionalInfo() { return this.additionalInfo; } + + public void setAdditionalInfo(String additionalInfo) { this.additionalInfo = additionalInfo; } + @Override public String getEventKey() { String key = user + "^" + accessType + "^" + resourcePath + "^" @@ -503,7 +510,7 @@ public class AuthzAuditEvent extends AuditEventBase { .append("tags=").append("[") .append(StringUtils.join(tags, ", ")) .append("]") - .append(FIELD_SEPARATOR); + .append(FIELD_SEPARATOR).append("additionalInfo=").append(additionalInfo); return sb; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2eb3c2fe/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java index ac9e93d..9a059b4 100644 --- a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java +++ b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java @@ -45,7 +45,7 @@ public class RangerHadoopConstants { public static final boolean RANGER_ADD_YARN_PERMISSION_DEFAULT = true ; // - // Loging constants + // Logging constants // public static final String AUDITLOG_FIELD_DELIMITER_PROP = "xasecure.auditlog.fieldDelimiterString"; public static final String AUDITLOG_RANGER_MODULE_ACL_NAME_PROP = "xasecure.auditlog.xasecureAcl.name" ; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2eb3c2fe/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java index bfb9126..66cee76 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java @@ -22,6 +22,7 @@ package org.apache.ranger.plugin.audit; import java.util.*; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.audit.model.AuthzAuditEvent; @@ -110,6 +111,7 @@ public class RangerDefaultAuditHandler implements RangerAccessResultProcessor { if (tags != null) { ret.setTags(tags); } + ret.setAdditionalInfo(getAdditionalInfo(request)); populateDefaults(ret); } @@ -221,4 +223,15 @@ public class RangerDefaultAuditHandler implements RangerAccessResultProcessor { return ret; } + + public String getAdditionalInfo(RangerAccessRequest request) { + if (StringUtils.isBlank(request.getRemoteIPAddress()) && CollectionUtils.isEmpty(request.getForwardedAddresses())) { + return null; + } + StringBuilder sb = new StringBuilder(); + sb.append("{\"remote-ip-address\":").append(request.getRemoteIPAddress()) + .append(", \"forwarded-ip-addresses\":[").append(StringUtils.join(request.getForwardedAddresses(), ", ")).append("]"); + + return sb.toString(); + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2eb3c2fe/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessRequest.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessRequest.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessRequest.java index 4a2aef8..749a1d4 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessRequest.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessRequest.java @@ -20,6 +20,7 @@ package org.apache.ranger.plugin.policyengine; import java.util.Date; +import java.util.List; import java.util.Map; import java.util.Set; @@ -40,6 +41,10 @@ public interface RangerAccessRequest { String getClientIPAddress(); + String getRemoteIPAddress(); + + List<String> getForwardedAddresses(); + String getClientType(); String getAction(); @@ -47,7 +52,7 @@ public interface RangerAccessRequest { String getRequestData(); String getSessionId(); - + Map<String, Object> getContext(); RangerAccessRequest getReadOnlyCopy(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2eb3c2fe/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessRequestImpl.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessRequestImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessRequestImpl.java index c4550c4..3bc8585 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessRequestImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessRequestImpl.java @@ -19,22 +19,29 @@ package org.apache.ranger.plugin.policyengine; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; - +import org.apache.log4j.Logger; public class RangerAccessRequestImpl implements RangerAccessRequest { + private static final Logger LOG = Logger.getLogger(RangerAccessRequestImpl.class); + private RangerAccessResource resource = null; private String accessType = null; private String user = null; private Set<String> userGroups = null; private Date accessTime = null; private String clientIPAddress = null; + private List<String> forwardedAddresses = null; + private String remoteIPAddress = null; private String clientType = null; private String action = null; private String requestData = null; @@ -54,10 +61,11 @@ public class RangerAccessRequestImpl implements RangerAccessRequest { setAccessType(accessType); setUser(user); setUserGroups(userGroups); + setForwardedAddresses(null); // set remaining fields to default value setAccessTime(null); - setClientIPAddress(null); + setRemoteIPAddress(null); setClientType(null); setAction(null); setRequestData(null); @@ -91,11 +99,17 @@ public class RangerAccessRequestImpl implements RangerAccessRequest { } @Override - public String getClientIPAddress() { - return clientIPAddress; + public String getClientIPAddress() { return clientIPAddress;} + + @Override + public String getRemoteIPAddress() { + return remoteIPAddress; } @Override + public List<String> getForwardedAddresses() { return forwardedAddresses; } + + @Override public String getClientType() { return clientType; } @@ -161,8 +175,16 @@ public class RangerAccessRequestImpl implements RangerAccessRequest { this.accessTime = (accessTime == null) ? new Date() : accessTime; } - public void setClientIPAddress(String clientIPAddress) { - this.clientIPAddress = clientIPAddress; + public void setClientIPAddress(String ipAddress) { + this.clientIPAddress = ipAddress; + } + + public void setForwardedAddresses(List<String> forwardedAddresses) { + this.forwardedAddresses = (forwardedAddresses == null) ? new ArrayList<String>() : forwardedAddresses; + } + + public void setRemoteIPAddress(String remoteIPAddress) { + this.remoteIPAddress = remoteIPAddress; } public void setClientType(String clientType) { @@ -187,6 +209,44 @@ public class RangerAccessRequestImpl implements RangerAccessRequest { this.context = (context == null) ? new HashMap<String, Object>() : context; } + protected void extractAndSetClientIPAddress(boolean useForwardedIPAddress, String[]trustedProxyAddresses) { + String ip = getRemoteIPAddress(); + if (ip == null) { + ip = getClientIPAddress(); + } + + String newIp = ip; + + if (useForwardedIPAddress) { + if (LOG.isDebugEnabled()) { + LOG.debug("Using X-Forward-For..."); + } + if (CollectionUtils.isNotEmpty(getForwardedAddresses())) { + if (trustedProxyAddresses != null && trustedProxyAddresses.length > 0) { + if (StringUtils.isNotEmpty(ip)) { + for (String trustedProxyAddress : trustedProxyAddresses) { + if (StringUtils.equals(ip, trustedProxyAddress)) { + newIp = getForwardedAddresses().get(0); + break; + } + } + } + } else { + newIp = getForwardedAddresses().get(0); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("No X-Forwarded-For addresses in the access-request"); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Old Remote/Client IP Address=" + ip + ", new IP Address=" + newIp); + } + setClientIPAddress(newIp); + } + @Override public String toString( ) { StringBuilder sb = new StringBuilder(); @@ -212,7 +272,9 @@ public class RangerAccessRequestImpl implements RangerAccessRequest { sb.append("} "); sb.append("accessTime={").append(accessTime).append("} "); - sb.append("clientIPAddress={").append(clientIPAddress).append("} "); + sb.append("clientIPAddress={").append(getClientIPAddress()).append("} "); + sb.append("forwardedAddresses={").append(StringUtils.join(forwardedAddresses, " ")).append("} "); + sb.append("remoteIPAddress={").append(remoteIPAddress).append("} "); sb.append("clientType={").append(clientType).append("} "); sb.append("action={").append(action).append("} "); sb.append("requestData={").append(requestData).append("} "); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2eb3c2fe/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessRequestReadOnly.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessRequestReadOnly.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessRequestReadOnly.java index 339aac5..a18e8bc 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessRequestReadOnly.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessRequestReadOnly.java @@ -21,6 +21,7 @@ package org.apache.ranger.plugin.policyengine; import java.util.Collections; import java.util.Date; +import java.util.List; import java.util.Map; import java.util.Set; @@ -30,6 +31,7 @@ public class RangerAccessRequestReadOnly implements RangerAccessRequest { // Cached here for reducing access overhead private final RangerAccessResource resource; private final Set<String> userGroups; + private final List<String> forwardedAddresses; private final Map<String, Object> context; RangerAccessRequestReadOnly(final RangerAccessRequest source) { @@ -37,6 +39,7 @@ public class RangerAccessRequestReadOnly implements RangerAccessRequest { this.resource = source.getResource().getReadOnlyCopy(); this.userGroups = Collections.unmodifiableSet(source.getUserGroups()); this.context = Collections.unmodifiableMap(source.getContext()); + this.forwardedAddresses = Collections.unmodifiableList(source.getForwardedAddresses()); } @Override @@ -64,6 +67,12 @@ public class RangerAccessRequestReadOnly implements RangerAccessRequest { public String getClientIPAddress() { return source.getClientIPAddress(); } @Override + public String getRemoteIPAddress() { return source.getRemoteIPAddress(); } + + @Override + public List<String> getForwardedAddresses() { return forwardedAddresses; } + + @Override public String getClientType() { return source.getClientType(); } @Override http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2eb3c2fe/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java index e5f1132..360da0c 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java @@ -67,5 +67,9 @@ public interface RangerPolicyEngine { boolean preCleanup(); + void setUseForwardedIPAddress(boolean useForwardedIPAddress); + + void setTrustedProxyAddresses(String[] trustedProxyAddresses); + void cleanup(); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2eb3c2fe/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java index e6e9a3a..3fdf3b6 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java @@ -51,6 +51,9 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { private List<RangerContextEnricher> allContextEnrichers; + private boolean useForwardedIPAddress = false; + private String[] trustedProxyAddresses = null; + public RangerPolicyEngineImpl(String appId, ServicePolicies servicePolicies, RangerPolicyEngineOptions options) { if (LOG.isDebugEnabled()) { LOG.debug("==> RangerPolicyEngineImpl(" + appId + ", " + servicePolicies + ", " + options + ")"); @@ -160,6 +163,9 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { } setResourceServiceDef(request); + if (request instanceof RangerAccessRequestImpl) { + ((RangerAccessRequestImpl) request).extractAndSetClientIPAddress(useForwardedIPAddress, trustedProxyAddresses); + } List<RangerContextEnricher> enrichers = allContextEnrichers; @@ -750,6 +756,16 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine { } @Override + public void setUseForwardedIPAddress(boolean useForwardedIPAddress) { + this.useForwardedIPAddress = useForwardedIPAddress; + } + + @Override + public void setTrustedProxyAddresses(String[] trustedProxyAddresses) { + this.trustedProxyAddresses = trustedProxyAddresses; + } + + @Override public String toString( ) { StringBuilder sb = new StringBuilder(); @@ -798,6 +814,8 @@ class RangerTagAccessRequest extends RangerAccessRequestImpl { super.setClientType(request.getClientType()); super.setClientIPAddress(request.getClientIPAddress()); + super.setRemoteIPAddress(request.getRemoteIPAddress()); + super.setForwardedAddresses(request.getForwardedAddresses()); super.setSessionId(request.getSessionId()); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2eb3c2fe/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java index 9b7cae8..d38a96c 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java @@ -40,6 +40,8 @@ import org.apache.ranger.plugin.util.ServicePolicies; public class RangerBasePlugin { private static final Log LOG = LogFactory.getLog(RangerBasePlugin.class); + public static final char RANGER_TRUSTED_PROXY_IPADDRESSES_SEPARATOR_CHAR = ','; + private String serviceType = null; private String appId = null; private String serviceName = null; @@ -47,6 +49,8 @@ public class RangerBasePlugin { private RangerPolicyEngine policyEngine = null; private RangerPolicyEngineOptions policyEngineOptions = new RangerPolicyEngineOptions(); private RangerAccessResultProcessor resultProcessor = null; + private boolean useForwardedIPAddress = false; + private String[] trustedProxyAddresses = null; Map<String, LogHistory> logHistoryList = new Hashtable<String, RangerBasePlugin.LogHistory>(); int logInterval = 30000; // 30 seconds @@ -90,9 +94,27 @@ public class RangerBasePlugin { String propertyPrefix = "ranger.plugin." + serviceType; long pollingIntervalMs = RangerConfiguration.getInstance().getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000); String cacheDir = RangerConfiguration.getInstance().get(propertyPrefix + ".policy.cache.dir"); - serviceName = RangerConfiguration.getInstance().get(propertyPrefix + ".service.name"); + useForwardedIPAddress = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".use.x-forwarded-for.ipaddress", false); + String trustedProxyAddressString = RangerConfiguration.getInstance().get(propertyPrefix + ".trusted.proxy.ipaddresses"); + trustedProxyAddresses = StringUtils.split(trustedProxyAddressString, RANGER_TRUSTED_PROXY_IPADDRESSES_SEPARATOR_CHAR); + if (trustedProxyAddresses != null) { + for (int i = 0; i < trustedProxyAddresses.length; i++) { + trustedProxyAddresses[i] = trustedProxyAddresses[i].trim(); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug(propertyPrefix + ".use.x-forwarded-for.ipaddress:" + useForwardedIPAddress); + LOG.debug(propertyPrefix + ".trusted.proxy.ipaddresses:[" + StringUtils.join(trustedProxyAddresses, ", ") + "]"); + } + + if (useForwardedIPAddress && StringUtils.isBlank(trustedProxyAddressString)) { + LOG.warn("Property " + propertyPrefix + ".use.x-forwarded-for.ipaddress" + " is set to true, and Property " + + propertyPrefix + ".trusted.proxy.ipaddresses" + " is not set"); + LOG.warn("Ranger plugin will trust RemoteIPAddress and treat first X-Forwarded-Address in the access-request as the clientIPAddress"); + } + policyEngineOptions.evaluatorType = RangerConfiguration.getInstance().get(propertyPrefix + ".policyengine.option.evaluator.type", RangerPolicyEvaluator.EVALUATOR_TYPE_AUTO); policyEngineOptions.cacheAuditResults = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.cache.audit.results", true); policyEngineOptions.disableContextEnrichers = RangerConfiguration.getInstance().getBoolean(propertyPrefix + ".policyengine.option.disable.context.enrichers", false); @@ -112,6 +134,8 @@ public class RangerBasePlugin { RangerPolicyEngine oldPolicyEngine = this.policyEngine; RangerPolicyEngine policyEngine = new RangerPolicyEngineImpl(appId, policies, policyEngineOptions); + policyEngine.setUseForwardedIPAddress(useForwardedIPAddress); + policyEngine.setTrustedProxyAddresses(trustedProxyAddresses); this.policyEngine = policyEngine; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2eb3c2fe/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java ---------------------------------------------------------------------- diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java index 6bb7486..dd15ff8 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java @@ -19,12 +19,18 @@ package org.apache.ranger.plugin.policyengine; -import com.google.gson.*; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; import com.google.gson.reflect.TypeToken; import org.apache.commons.lang.StringUtils; import org.apache.ranger.audit.provider.AuditHandler; import org.apache.ranger.audit.provider.AuditProviderFactory; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerTag; @@ -37,7 +43,12 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import java.io.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; import java.lang.reflect.Type; import java.util.List; import java.util.Map; @@ -48,8 +59,7 @@ import static org.junit.Assert.*; public class TestPolicyEngine { static RangerPolicyEngine policyEngine = null; - static Gson gsonBuilder = null; - + static Gson gsonBuilder = null; @BeforeClass public static void setUpBeforeClass() throws Exception { @@ -96,18 +106,15 @@ public class TestPolicyEngine { System.out.println("provider=" + provider.toString()); -/* - // For setting up TestTagProvider - - Path filePath = new Path("file:///tmp/ranger-admin-test-site.xml"); - Configuration config = new Configuration(); - - FileSystem fs = filePath.getFileSystem(config); - - FSDataOutputStream outStream = fs.create(filePath, true); + File file = File.createTempFile("ranger-admin-test-site", ".xml") ; + file.deleteOnExit(); + FileOutputStream outStream = new FileOutputStream(file); OutputStreamWriter writer = new OutputStreamWriter(outStream); + /* + // For setting up TestTagProvider + writer.write("<configuration>\n" + " <property>\n" + " <name>ranger.plugin.tag.policy.rest.url</name>\n" + @@ -118,13 +125,34 @@ public class TestPolicyEngine { " <value>http://os-def:6080</value>\n" + " </property>\n" + "</configuration>\n"); + */ + writer.write("<configuration>\n" + + /* + // For setting up TestTagProvider + " <property>\n" + + " <name>ranger.plugin.tag.policy.rest.url</name>\n" + + " <value>http://os-def:6080</value>\n" + + " </property>\n" + + " <property>\n" + + " <name>ranger.externalurl</name>\n" + + " <value>http://os-def:6080</value>\n" + + " </property>\n" + + */ + // For setting up x-forwarded-for for Hive + " <property>\n" + + " <name>ranger.plugin.hive.use.x-forwarded-for.ipaddress</name>\n" + + " <value>true</value>\n" + + " </property>\n" + + " <property>\n" + + " <name>ranger.plugin.hive.trusted.proxy.ipaddresses</name>\n" + + " <value>255.255.255.255; 128.101.101.101;128.101.101.99</value>\n" + + " </property>\n" + + "</configuration>\n"); writer.close(); - RangerConfiguration rangerConfig = RangerConfiguration.getInstance(); - rangerConfig.addResource(filePath); -*/ - + RangerConfiguration config = RangerConfiguration.getInstance(); + config.addResource(new org.apache.hadoop.fs.Path(file.toURI())); } @AfterClass @@ -210,7 +238,7 @@ public class TestPolicyEngine { private void runTestsFromResourceFiles(String[] resourceNames) { for(String resourceName : resourceNames) { - InputStream inStream = this.getClass().getResourceAsStream(resourceName); + InputStream inStream = this.getClass().getResourceAsStream(resourceName); InputStreamReader reader = new InputStreamReader(inStream); runTests(reader, resourceName); @@ -223,7 +251,7 @@ public class TestPolicyEngine { assertTrue("invalid input: " + testName, testCase != null && testCase.serviceDef != null && testCase.policies != null && testCase.tests != null); ServicePolicies servicePolicies = new ServicePolicies(); - servicePolicies.setServiceName(testCase.serviceName);; + servicePolicies.setServiceName(testCase.serviceName); servicePolicies.setServiceDef(testCase.serviceDef); servicePolicies.setPolicies(testCase.policies); @@ -240,26 +268,38 @@ public class TestPolicyEngine { policyEngineOptions.disableTagPolicyEvaluation = false; + boolean useForwardedIPAddress = RangerConfiguration.getInstance().getBoolean("ranger.plugin.hive.use.x-forwarded-for.ipaddress", false); + String trustedProxyAddressString = RangerConfiguration.getInstance().get("ranger.plugin.hive.trusted.proxy.ipaddresses"); + String[] trustedProxyAddresses = StringUtils.split(trustedProxyAddressString, ';'); + if (trustedProxyAddresses != null) { + for (int i = 0; i < trustedProxyAddresses.length; i++) { + trustedProxyAddresses[i] = trustedProxyAddresses[i].trim(); + } + } policyEngine = new RangerPolicyEngineImpl(testName, servicePolicies, policyEngineOptions); + policyEngine.setUseForwardedIPAddress(useForwardedIPAddress); + policyEngine.setTrustedProxyAddresses(trustedProxyAddresses); RangerAccessRequest request = null; for(TestData test : testCase.tests) { - if (test.request.getContext().containsKey(RangerAccessRequestUtil.KEY_CONTEXT_TAGS) || - test.request.getContext().containsKey(RangerAccessRequestUtil.KEY_CONTEXT_REQUESTED_RESOURCES)) { + request = test.request; + if (request.getContext().containsKey(RangerAccessRequestUtil.KEY_CONTEXT_TAGS) || + request.getContext().containsKey(RangerAccessRequestUtil.KEY_CONTEXT_REQUESTED_RESOURCES)) { // Create a new AccessRequest RangerAccessRequestImpl newRequest = - new RangerAccessRequestImpl(test.request.getResource(), test.request.getAccessType(), - test.request.getUser(), test.request.getUserGroups()); - - newRequest.setClientType(test.request.getClientType()); - newRequest.setAccessTime(test.request.getAccessTime()); - newRequest.setAction(test.request.getAction()); - newRequest.setClientIPAddress(test.request.getClientIPAddress()); - newRequest.setRequestData(test.request.getRequestData()); - newRequest.setSessionId(test.request.getSessionId()); - - Map<String, Object> context = test.request.getContext(); + new RangerAccessRequestImpl(request.getResource(), request.getAccessType(), + request.getUser(), request.getUserGroups()); + + newRequest.setClientType(request.getClientType()); + newRequest.setAccessTime(request.getAccessTime()); + newRequest.setAction(request.getAction()); + newRequest.setRemoteIPAddress(request.getRemoteIPAddress()); + newRequest.setForwardedAddresses(request.getForwardedAddresses()); + newRequest.setRequestData(request.getRequestData()); + newRequest.setSessionId(request.getSessionId()); + + Map<String, Object> context = request.getContext(); String tagsJsonString = (String) context.get(RangerAccessRequestUtil.KEY_CONTEXT_TAGS); context.remove(RangerAccessRequestUtil.KEY_CONTEXT_TAGS); @@ -274,7 +314,7 @@ public class TestPolicyEngine { System.err.println("TestPolicyEngine.runTests(): error parsing TAGS JSON string in file " + testName + ", tagsJsonString=" + tagsJsonString + ", exception=" + e); } - } else if (test.request.getContext().containsKey(RangerAccessRequestUtil.KEY_CONTEXT_REQUESTED_RESOURCES)) { + } else if (request.getContext().containsKey(RangerAccessRequestUtil.KEY_CONTEXT_REQUESTED_RESOURCES)) { String resourcesJsonString = (String) context.get(RangerAccessRequestUtil.KEY_CONTEXT_REQUESTED_RESOURCES); context.remove(RangerAccessRequestUtil.KEY_CONTEXT_REQUESTED_RESOURCES); if (!StringUtils.isEmpty(resourcesJsonString)) { @@ -303,16 +343,13 @@ public class TestPolicyEngine { // presence of tags!!! // Safe cast - RangerAccessResourceImpl accessResource = (RangerAccessResourceImpl) test.request.getResource(); + RangerAccessResourceImpl accessResource = (RangerAccessResourceImpl) request.getResource(); accessResource.setServiceDef(testCase.serviceDef); request = newRequest; } else - if (test.request.getContext().containsKey(RangerAccessRequestUtil.KEY_CONTEXT_REQUESTED_RESOURCES)) { - } - else { - request = test.request; + if (!request.getContext().containsKey(RangerAccessRequestUtil.KEY_CONTEXT_REQUESTED_RESOURCES)) { policyEngine.preProcess(request); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2eb3c2fe/agents-common/src/test/resources/policyengine/test_policyengine_geo.json ---------------------------------------------------------------------- diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_geo.json b/agents-common/src/test/resources/policyengine/test_policyengine_geo.json index eab1223..4249996 100644 --- a/agents-common/src/test/resources/policyengine/test_policyengine_geo.json +++ b/agents-common/src/test/resources/policyengine/test_policyengine_geo.json @@ -76,8 +76,9 @@ {"name":"ALLOW 'read /finance/restricted/sales.db' for g=finance; valid clientIPAddress", "request":{ "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, - "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/sales.db", - "clientIPAddress":"255.255.255.255" + "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/sales.db" + ,"remoteIPAddress":"255.255.255.255" + ,"forwardedAddresses":["1.0.32.1"] }, "result":{"isAudited":true,"isAllowed":true,"policyId":3} } @@ -86,7 +87,7 @@ "request":{ "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/sales.db", - "clientIPAddress":"128.101.101.99" + "remoteIPAddress":"128.101.101.99" }, "result":{"isAudited":true,"isAllowed":false,"policyId":-1} } @@ -103,7 +104,7 @@ "request":{ "resource":{"elements":{"path":"/finance/restricted/hr/payroll.db"}}, "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/hr/payroll.db", - "clientIPAddress":"128.101.101.101" + "remoteIPAddress":"128.101.101.101" }, "result":{"isAudited":true,"isAllowed":true,"policyId":3} @@ -113,7 +114,7 @@ "request":{ "resource":{"elements":{"path":"/operations/visitors.db"}}, "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /operations/visitors.db", - "clientIPAddress":"128.101.101.99" + "remoteIPAddress":"128.101.101.99" }, "result":{"isAudited":false,"isAllowed":false,"policyId":-1} } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2eb3c2fe/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json ---------------------------------------------------------------------- diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json b/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json index a7f355c..ea167f4 100644 --- a/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json +++ b/agents-common/src/test/resources/policyengine/test_policyengine_hdfs.json @@ -68,7 +68,7 @@ "request":{ "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/sales.db", - "clientIPAddress":"255.255.255.255" + "remoteIPAddress":"255.255.255.255" }, "result":{"isAudited":true,"isAllowed":true,"policyId":3} } @@ -77,7 +77,7 @@ "request":{ "resource":{"elements":{"path":"/finance/restricted/sales.db"}}, "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/sales.db", - "clientIPAddress":"128.101.101.99" + "remoteIPAddress":"128.101.101.99" }, "result":{"isAudited":true,"isAllowed":false,"policyId":-1} } @@ -94,7 +94,7 @@ "request":{ "resource":{"elements":{"path":"/finance/restricted/hr/payroll.db"}}, "accessType":"read","user":"user1","userGroups":["finance"],"requestData":"read /finance/restricted/hr/payroll.db", - "clientIPAddress":"128.101.101.101" + "remoteIPAddress":"128.101.101.101" }, "result":{"isAudited":true,"isAllowed":true,"policyId":3} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2eb3c2fe/agents-common/src/test/resources/policyengine/test_policyengine_hive.json ---------------------------------------------------------------------- diff --git a/agents-common/src/test/resources/policyengine/test_policyengine_hive.json b/agents-common/src/test/resources/policyengine/test_policyengine_hive.json index 4151a87..b7f9888 100644 --- a/agents-common/src/test/resources/policyengine/test_policyengine_hive.json +++ b/agents-common/src/test/resources/policyengine/test_policyengine_hive.json @@ -59,6 +59,7 @@ "request":{ "resource":{"elements":{"database":"db1", "table":"tmp", "column":"tmp_1"}}, "accessType":"select","user":"user1","userGroups":["users"],"requestData":"select tmp_1 from db1.tmp for user1" + ,"remoteIPAddress":"1.1.1.1","forwardedAddresses":["127.0.0.1","10.10.10.10"] }, "result":{"isAudited":false,"isAllowed":false,"policyId":-1} }
