RANGER-561: YARN plugin updated to fallback to YARN ACL, instead of creating Ranger policies
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/a31d901c Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/a31d901c Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/a31d901c Branch: refs/heads/tag-policy Commit: a31d901cc0cbc608550e1c99c60c1fb6e7d13388 Parents: 088f3cf Author: Madhan Neethiraj <[email protected]> Authored: Fri Jun 19 17:34:13 2015 -0400 Committer: Madhan Neethiraj <[email protected]> Committed: Thu Jun 25 08:04:23 2015 -0700 ---------------------------------------------------------------------- .../hadoop/constants/RangerHadoopConstants.java | 5 + .../yarn/authorizer/RangerYarnAuthorizer.java | 208 +++++++++++++------ 2 files changed, 148 insertions(+), 65 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a31d901c/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java index a800027..ac9e93d 100644 --- a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java +++ b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java @@ -41,17 +41,22 @@ public class RangerHadoopConstants { public static final String STORM_ACCESS_VERIFIER_CLASS_NAME_PROP = "storm.authorization.verifier.classname" ; public static final String STORM_ACCESS_VERIFIER_CLASS_NAME_DEFAULT_VALUE = "org.apache.ranger.pdp.storm.RangerAuthorizer" ; + public static final String RANGER_ADD_YARN_PERMISSION_PROP = "ranger.add-yarn-authorization" ; + public static final boolean RANGER_ADD_YARN_PERMISSION_DEFAULT = true ; + // // Loging constants // public static final String AUDITLOG_FIELD_DELIMITER_PROP = "xasecure.auditlog.fieldDelimiterString"; public static final String AUDITLOG_RANGER_MODULE_ACL_NAME_PROP = "xasecure.auditlog.xasecureAcl.name" ; public static final String AUDITLOG_HADOOP_MODULE_ACL_NAME_PROP = "xasecure.auditlog.hadoopAcl.name" ; + public static final String AUDITLOG_YARN_MODULE_ACL_NAME_PROP = "ranger.auditlog.yarnAcl.name" ; public static final String DEFAULT_LOG_FIELD_DELIMITOR = "|" ; public static final String DEFAULT_XASECURE_MODULE_ACL_NAME = "xasecure-acl" ; public static final String DEFAULT_RANGER_MODULE_ACL_NAME = "ranger-acl" ; public static final String DEFAULT_HADOOP_MODULE_ACL_NAME = "hadoop-acl" ; + public static final String DEFAULT_YARN_MODULE_ACL_NAME = "yarn-acl" ; public static final String AUDITLOG_FIELDINFO_VISIBLE_PROP = "xasecure.auditlog.fieldInfoVisible" ; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a31d901c/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java ---------------------------------------------------------------------- diff --git a/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java b/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java index e322477..ab9b7a9 100644 --- a/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java +++ b/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java @@ -21,13 +21,9 @@ package org.apache.ranger.authorization.yarn.authorizer; import java.net.InetAddress; -import java.util.Collection; -import java.util.HashSet; +import java.util.HashMap; import java.util.Map; -import java.util.Set; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -35,14 +31,16 @@ import org.apache.hadoop.ipc.Server; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.security.*; +import org.apache.hadoop.yarn.security.PrivilegedEntity.EntityType; +import org.apache.ranger.audit.model.AuthzAuditEvent; +import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; +import org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants; import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler; import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; import org.apache.ranger.plugin.policyengine.RangerAccessResult; -import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; import org.apache.ranger.plugin.service.RangerBasePlugin; -import org.apache.ranger.plugin.util.GrantRevokeRequest; import com.google.common.collect.Sets; @@ -51,11 +49,14 @@ public class RangerYarnAuthorizer extends YarnAuthorizationProvider { public static final String ACCESS_TYPE_SUBMIT_APP = "submit-app"; public static final String ACCESS_TYPE_ADMIN = "admin"; + private static boolean yarnAuthEnabled = RangerHadoopConstants.RANGER_ADD_YARN_PERMISSION_DEFAULT; + private static final Log LOG = LogFactory.getLog(RangerYarnAuthorizer.class); private static volatile RangerYarnPlugin yarnPlugin = null; private AccessControlList admins = null; + private Map<PrivilegedEntity, Map<AccessType, AccessControlList>> yarnAcl = new HashMap<PrivilegedEntity, Map<AccessType, AccessControlList>>(); @Override public void init(Configuration conf) { @@ -78,6 +79,8 @@ public class RangerYarnAuthorizer extends YarnAuthorizationProvider { } } + RangerYarnAuthorizer.yarnAuthEnabled = RangerConfiguration.getInstance().getBoolean(RangerHadoopConstants.RANGER_ADD_YARN_PERMISSION_PROP, RangerHadoopConstants.RANGER_ADD_YARN_PERMISSION_DEFAULT); + if(LOG.isDebugEnabled()) { LOG.debug("<== RangerYarnAuthorizer.init()"); } @@ -86,23 +89,34 @@ public class RangerYarnAuthorizer extends YarnAuthorizationProvider { @Override public boolean checkPermission(AccessType accessType, PrivilegedEntity entity, UserGroupInformation ugi) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerYarnAuthorizer.checkPermission(" + accessType + ", " + entity + ", " + ugi + ")"); + LOG.debug("==> RangerYarnAuthorizer.checkPermission(" + accessType + ", " + toString(entity) + ", " + ugi + ")"); } - boolean ret = false; - - RangerYarnPlugin plugin = yarnPlugin; + boolean ret = false; + RangerYarnPlugin plugin = yarnPlugin; + RangerYarnAuditHandler auditHandler = null; + RangerAccessResult result = null; if(plugin != null) { RangerYarnAccessRequest request = new RangerYarnAccessRequest(entity, getRangerAccessType(accessType), accessType.name(), ugi); - RangerAccessResult result = plugin.isAccessAllowed(request); + auditHandler = new RangerYarnAuditHandler(); + result = plugin.isAccessAllowed(request, auditHandler); + } + + if(RangerYarnAuthorizer.yarnAuthEnabled && (result == null || !result.getIsAccessDetermined())) { + ret = isAllowedByYarnAcl(accessType, entity, ugi, auditHandler); + } else { ret = result == null ? false : result.getIsAllowed(); } + if(auditHandler != null) { + auditHandler.flushAudit(); + } + if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerYarnAuthorizer.checkPermission(" + accessType + ", " + entity + ", " + ugi + "): " + ret); + LOG.debug("<== RangerYarnAuthorizer.checkPermission(" + accessType + ", " + toString(entity) + ", " + ugi + "): " + ret); } return ret; @@ -116,6 +130,8 @@ public class RangerYarnAuthorizer extends YarnAuthorizationProvider { boolean ret = false; + AccessControlList admins = this.admins; + if(admins != null) { ret = admins.isUserAllowed(ugi); } @@ -143,58 +159,51 @@ public class RangerYarnAuthorizer extends YarnAuthorizationProvider { @Override public void setPermission(PrivilegedEntity entity, Map<AccessType, AccessControlList> permission, UserGroupInformation ugi) { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerYarnAuthorizer.setPermission(" + entity + ", " + permission + ", " + ugi + ")"); + LOG.debug("==> RangerYarnAuthorizer.setPermission(" + toString(entity) + ", " + permission + ", " + ugi + ")"); } - RangerYarnPlugin plugin = yarnPlugin; + yarnAcl.put(entity, permission); - if(plugin != null && entity != null && !MapUtils.isEmpty(permission) && ugi != null) { - RangerYarnResource resource = new RangerYarnResource(entity); - - GrantRevokeRequest request = new GrantRevokeRequest(); - request.setResource(resource.getAsMap()); - request.setGrantor(ugi.getShortUserName()); - request.setDelegateAdmin(Boolean.FALSE); - request.setEnableAudit(Boolean.TRUE); - request.setReplaceExistingPermissions(Boolean.FALSE); - request.setIsRecursive(Boolean.TRUE); - - for(Map.Entry<AccessType, AccessControlList> e : permission.entrySet()) { - AccessType accessType = e.getKey(); - AccessControlList acl = e.getValue(); - - Set<String> accessTypes = new HashSet<String>(); - accessTypes.add(getRangerAccessType(accessType)); - request.setAccessTypes(accessTypes); - - if(acl.isAllAllowed()) { - Set<String> publicGroup = new HashSet<String>(); - publicGroup.add(RangerPolicyEngine.GROUP_PUBLIC); - - request.setUsers(null); - request.setGroups(publicGroup); - } else if(CollectionUtils.isEmpty(acl.getUsers()) && CollectionUtils.isEmpty(acl.getGroups())) { - if(LOG.isDebugEnabled()) { - LOG.debug("grantAccess(): empty users and groups - skipped"); - } - - continue; - } else { - request.setUsers(getSet(acl.getUsers())); - request.setGroups(getSet(acl.getGroups())); - } + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerYarnAuthorizer.setPermission(" + toString(entity) + ", " + permission + ", " + ugi + ")"); + } + } - try { - plugin.grantAccess(request, plugin.getResultProcessor()); - } catch(Exception excp) { - LOG.error("grantAccess(" + request + ") failed", excp); - } + public boolean isAllowedByYarnAcl(AccessType accessType, PrivilegedEntity entity, UserGroupInformation ugi, RangerYarnAuditHandler auditHandler) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerYarnAuthorizer.isAllowedByYarnAcl(" + accessType + ", " + toString(entity) + ", " + ugi + ")"); + } + + boolean ret = false; + + for(Map.Entry<PrivilegedEntity, Map<AccessType, AccessControlList>> e : yarnAcl.entrySet()) { + PrivilegedEntity aclEntity = e.getKey(); + Map<AccessType, AccessControlList> entityPermissions = e.getValue(); + + AccessControlList acl = entityPermissions == null ? null : entityPermissions.get(accessType); + + if(acl == null || !acl.isUserAllowed(ugi)) { + continue; } + + if(! isSelfOrChildOf(entity, aclEntity)) { + continue; + } + + ret = true; + + break; + } + + if(auditHandler != null) { + auditHandler.logYarnAclEvent(ret); } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerYarnAuthorizer.setPermission(" + entity + ", " + permission + ", " + ugi + ")"); + LOG.debug("<== RangerYarnAuthorizer.isAllowedByYarnAcl(" + accessType + ", " + toString(entity) + ", " + ugi + "): " + ret); } + + return ret; } private static String getRangerAccessType(AccessType accessType) { @@ -213,22 +222,32 @@ public class RangerYarnAuthorizer extends YarnAuthorizationProvider { return ret; } - private Set<String> getSet(Collection<String> strings) { - Set<String> ret = null; + private boolean isSelfOrChildOf(PrivilegedEntity queue, PrivilegedEntity parentQueue) { + boolean ret = queue.equals(parentQueue); + + if(!ret && queue.getType() == EntityType.QUEUE) { + String queueName = queue.getName(); + String parentQueueName = parentQueue.getName(); - if(! CollectionUtils.isEmpty(strings)) { - if(strings instanceof Set<?>) { - ret = (Set<String>)strings; - } else { - ret = new HashSet<String>(); - for(String str : strings) { - ret.add(str); + if(queueName.contains(".") && !StringUtil.isEmpty(parentQueueName)) { + if(parentQueueName.charAt(parentQueueName.length() - 1) != '.') { + parentQueueName += "."; } + + ret = queueName.startsWith(parentQueueName); } } return ret; } + + private String toString(PrivilegedEntity entity) { + if(entity != null) { + return "{name=" + entity.getName() + "; type=" + entity.getType() + "}"; + } + + return "null"; + } } class RangerYarnPlugin extends RangerBasePlugin { @@ -273,4 +292,63 @@ class RangerYarnAccessRequest extends RangerAccessRequestImpl { } return ret ; } +} + +class RangerYarnAuditHandler extends RangerDefaultAuditHandler { + private static final Log LOG = LogFactory.getLog(RangerYarnAuditHandler.class); + + private static final String YarnModuleName = RangerConfiguration.getInstance().get(RangerHadoopConstants.AUDITLOG_YARN_MODULE_ACL_NAME_PROP , RangerHadoopConstants.DEFAULT_YARN_MODULE_ACL_NAME) ; + + private boolean isAuditEnabled = false; + private AuthzAuditEvent auditEvent = null; + + public RangerYarnAuditHandler() { + } + + @Override + public void processResult(RangerAccessResult result) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerYarnAuditHandler.logAudit(" + result + ")"); + } + + if(! isAuditEnabled && result.getIsAudited()) { + isAuditEnabled = true; + } + + auditEvent = super.getAuthzEvents(result); + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerYarnAuditHandler.logAudit(" + result + "): " + auditEvent); + } + } + + public void logYarnAclEvent(boolean accessGranted) { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerYarnAuditHandler.logYarnAclEvent(" + accessGranted + ")"); + } + + if(auditEvent != null) { + auditEvent.setAccessResult((short) (accessGranted ? 1 : 0)); + auditEvent.setAclEnforcer(YarnModuleName); + auditEvent.setPolicyId(-1); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerYarnAuditHandler.logYarnAclEvent(" + accessGranted + "): " + auditEvent); + } + } + + public void flushAudit() { + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerYarnAuditHandler.flushAudit(" + isAuditEnabled + ", " + auditEvent + ")"); + } + + if(isAuditEnabled) { + super.logAuthzAudit(auditEvent); + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerYarnAuditHandler.flushAudit(" + isAuditEnabled + ", " + auditEvent + ")"); + } + } } \ No newline at end of file
