This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 4c2540e765b1682bdcf839aafa3ff318b47762ee Author: Ramesh Mani <[email protected]> AuthorDate: Mon Sep 28 16:06:20 2020 -0700 RANGER-3048: updated HDFS plugin to include user action (like list/mkdir/delete) in audit logs --- .../hadoop/constants/RangerHadoopConstants.java | 5 + .../plugin/util/RangerAccessRequestUtil.java | 1 + .../authorization/hadoop/RangerHdfsAuthorizer.java | 126 ++++++++++++++++++++- .../services/hdfs/RangerHdfsAuthorizerTest.java | 59 ++++++++-- .../src/main/webapp/scripts/utils/XAViewUtils.js | 5 +- 5 files changed, 182 insertions(+), 14 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java index 6d9fe26..31e4c0f 100644 --- a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java +++ b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java @@ -28,6 +28,11 @@ public class RangerHadoopConstants { public static final String WRITE_ACCCESS_TYPE = "write"; public static final String EXECUTE_ACCCESS_TYPE = "execute"; + public static final String READ_EXECUTE_PERM = "READ_EXECUTE"; + public static final String WRITE_EXECUTE_PERM = "WRITE_EXECUTE"; + public static final String READ_WRITE_PERM = "READ_WRITE"; + public static final String ALL_PERM = "ALL"; + public static final String HDFS_ROOT_FOLDER_PATH_ALT = ""; public static final String HDFS_ROOT_FOLDER_PATH = "/"; diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerAccessRequestUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerAccessRequestUtil.java index bc52bdb..a22027a 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerAccessRequestUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerAccessRequestUtil.java @@ -43,6 +43,7 @@ public class RangerAccessRequestUtil { public static final String KEY_USER = "USER"; public static final String KEY_OWNER = "OWNER"; public static final String KEY_ROLES = "ROLES"; + public static final String KEY_CONTEXT_ACCESSTYPES = "ACCESSTYPES"; public static void setRequestTagsInContext(Map<String, Object> context, Set<RangerTagForEval> tags) { if(CollectionUtils.isEmpty(tags)) { diff --git a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java index d8bcac7..44fec7b 100644 --- a/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java +++ b/hdfs-agent/src/main/java/org/apache/ranger/authorization/hadoop/RangerHdfsAuthorizer.java @@ -23,11 +23,16 @@ import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConst import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants.HDFS_ROOT_FOLDER_PATH; import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants.READ_ACCCESS_TYPE; import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants.WRITE_ACCCESS_TYPE; +import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants.READ_EXECUTE_PERM; +import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants.WRITE_EXECUTE_PERM; +import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants.READ_WRITE_PERM; +import static org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants.ALL_PERM; import java.net.InetAddress; import java.security.SecureRandom; import java.util.*; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; @@ -42,6 +47,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeAttributes; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.util.ReadOnlyList; +import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; @@ -219,24 +225,47 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider { } @Override + public void checkPermissionWithContext(AuthorizationContext authzContext) throws AccessControlException { + checkRangerPermission(authzContext.getFsOwner(), authzContext.getSupergroup(), + authzContext.getCallerUgi(), authzContext.getInodeAttrs(), + authzContext.getInodes(), authzContext.getPathByNameArr(), + authzContext.getSnapshotId(), authzContext.getPath(), + authzContext.getAncestorIndex(), authzContext.isDoCheckOwner(), + authzContext.getAncestorAccess(), authzContext.getParentAccess(), + authzContext.getAccess(), authzContext.getSubAccess(), + authzContext.isIgnoreEmptyDir(), authzContext.getOperationName(), + authzContext.getCallerContext()); + } + + @Override public void checkPermission(String fsOwner, String superGroup, UserGroupInformation ugi, INodeAttributes[] inodeAttrs, INode[] inodes, byte[][] pathByNameArr, int snapshotId, String path, int ancestorIndex, boolean doCheckOwner, FsAction ancestorAccess, FsAction parentAccess, FsAction access, FsAction subAccess, boolean ignoreEmptyDir) throws AccessControlException { + checkRangerPermission(fsOwner, superGroup, ugi, inodeAttrs, inodes, pathByNameArr, snapshotId, path, ancestorIndex, doCheckOwner, ancestorAccess, parentAccess, access, subAccess, ignoreEmptyDir, null, null); + } + + private void checkRangerPermission(String fsOwner, String superGroup, UserGroupInformation ugi, + INodeAttributes[] inodeAttrs, INode[] inodes, byte[][] pathByNameArr, + int snapshotId, String path, int ancestorIndex, boolean doCheckOwner, + FsAction ancestorAccess, FsAction parentAccess, FsAction access, + FsAction subAccess, boolean ignoreEmptyDir, String operationName, CallerContext callerContext ) throws AccessControlException { AuthzStatus authzStatus = AuthzStatus.NOT_DETERMINED; RangerHdfsPlugin plugin = rangerPlugin; RangerHdfsAuditHandler auditHandler = null; String user = ugi != null ? ugi.getShortUserName() : null; Set<String> groups = ugi != null ? Sets.newHashSet(ugi.getGroupNames()) : null; String resourcePath = path; + String callerctxt = callerContext != null ? callerContext.toString() : null; if(LOG.isDebugEnabled()) { LOG.debug("==> RangerAccessControlEnforcer.checkPermission(" + "fsOwner=" + fsOwner + "; superGroup=" + superGroup + ", inodesCount=" + (inodes != null ? inodes.length : 0) + ", snapshotId=" + snapshotId + ", user=" + user + ", provided-path=" + path + ", ancestorIndex=" + ancestorIndex + ", doCheckOwner="+ doCheckOwner + ", ancestorAccess=" + ancestorAccess + ", parentAccess=" + parentAccess - + ", access=" + access + ", subAccess=" + subAccess + ", ignoreEmptyDir=" + ignoreEmptyDir + ")"); + + ", access=" + access + ", subAccess=" + subAccess + ", ignoreEmptyDir=" + ignoreEmptyDir + ", operationName=" + operationName + + ", callerContext=" + callerctxt +")"); } RangerPerfTracer perf = null; @@ -320,7 +349,7 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider { parent = inodes.length > 1 ? inodes[inodes.length - 2] : null; inode = inodes[inodes.length - 1]; // could be null while creating a new file - auditHandler = doNotGenerateAuditRecord ? null : new RangerHdfsAuditHandler(resourcePath, isTraverseOnlyCheck, rangerPlugin.getHadoopModuleName(), rangerPlugin.getExcludedUsers()); + auditHandler = doNotGenerateAuditRecord ? null : new RangerHdfsAuditHandler(resourcePath, isTraverseOnlyCheck, rangerPlugin.getHadoopModuleName(), rangerPlugin.getExcludedUsers(), operationName, callerctxt); /* Hadoop versions prior to 2.8.0 didn't ask for authorization of parent/ancestor traversal for * reading or writing a file. However, Hadoop version 2.8.0 and later ask traversal authorization for @@ -671,6 +700,9 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider { for(String accessType : accessTypes) { RangerHdfsAccessRequest request = new RangerHdfsAccessRequest(inode, path, pathOwner, access, accessType, user, groups); + Map<String, Object> requestContext = request.getContext(); + requestContext.put(RangerAccessRequestUtil.KEY_CONTEXT_ACCESSTYPES, accessTypes); + RangerAccessResult result = plugin.isAccessAllowed(request, auditHandler); if (result == null || !result.getIsAccessDetermined()) { @@ -732,6 +764,9 @@ public class RangerHdfsAuthorizer extends INodeAttributeProvider { for (String accessType : accessTypes) { RangerHdfsAccessRequest request = new RangerHdfsAccessRequest(null, subDirPath, pathOwner, access, accessType, user, groups); + Map<String, Object> requestContext = request.getContext(); + requestContext.put(RangerAccessRequestUtil.KEY_CONTEXT_ACCESSTYPES, accessTypes); + RangerAccessResult result = plugin.isAccessAllowed(request, null); if (result == null || !result.getIsAccessDetermined()) { @@ -900,12 +935,16 @@ class RangerHdfsAuditHandler extends RangerDefaultAuditHandler { private final String hadoopModuleName; private final Set<String> excludeUsers; + private final String operationName; + private final String callerContext; - public RangerHdfsAuditHandler(String pathToBeValidated, boolean auditOnlyIfDenied, String hadoopModuleName, Set<String> excludedUsers) { + public RangerHdfsAuditHandler(String pathToBeValidated,boolean auditOnlyIfDenied, String hadoopModuleName, Set<String> excludedUsers, String operationName, String callerContext) { this.pathToBeValidated = pathToBeValidated; this.auditOnlyIfDenied = auditOnlyIfDenied; this.hadoopModuleName = hadoopModuleName; this.excludeUsers = excludedUsers; + this.operationName = operationName; + this.callerContext = callerContext; } @Override @@ -937,6 +976,10 @@ class RangerHdfsAuditHandler extends RangerDefaultAuditHandler { auditEvent.setAccessResult((short) (result.getIsAllowed() ? 1 : 0)); auditEvent.setPolicyId(result.getPolicyId()); auditEvent.setPolicyVersion(result.getPolicyVersion()); + addOperationNameToAuditEvent(); + + auditEvent.setAction(getAccessType(request.getAccessType())); + auditEvent.setAdditionalInfo(getAdditionalInfo(request)); Set<String> tags = getTags(request); if (tags != null) { @@ -950,6 +993,22 @@ class RangerHdfsAuditHandler extends RangerDefaultAuditHandler { } } + @Override + public String getAdditionalInfo(RangerAccessRequest request) { + StringBuilder sb = null; + String additionalInfo = super.getAdditionalInfo(request); + if (additionalInfo == null) { + sb = new StringBuilder(""); + } else { + sb = new StringBuilder(additionalInfo); + } + String accessTypes = getAccessTypesAsString(request); + if (accessTypes != null) { + sb.append(", \"accessTypes\":[").append(accessTypes).append("]"); + } + return sb.toString(); + } + public void logHadoopEvent(String path, FsAction action, boolean accessGranted) { if(LOG.isDebugEnabled()) { LOG.debug("==> RangerHdfsAuditHandler.logHadoopEvent(" + path + ", " + action + ", " + accessGranted + ")"); @@ -958,9 +1017,14 @@ class RangerHdfsAuditHandler extends RangerDefaultAuditHandler { if(auditEvent != null) { auditEvent.setResultReason(path); auditEvent.setAccessResult((short) (accessGranted ? 1 : 0)); - auditEvent.setAccessType(action == null ? null : action.toString()); + String accessType = (action == null) ? null : action.toString(); + auditEvent.setAccessType(accessType); auditEvent.setAclEnforcer(hadoopModuleName); auditEvent.setPolicyId(-1); + if (accessType != null) { + auditEvent.setAction(getAccessType(accessType)); + } + addOperationNameToAuditEvent(); } if(LOG.isDebugEnabled()) { @@ -987,5 +1051,59 @@ class RangerHdfsAuditHandler extends RangerDefaultAuditHandler { LOG.debug("<== RangerHdfsAuditHandler.flushAudit(" + isAuditEnabled + ", " + auditEvent + ")"); } } + + private String getAccessType(String accessType) { + String ret = accessType; + + switch (accessType) { + case READ_EXECUTE_PERM: + ret = READ_ACCCESS_TYPE; + break; + case WRITE_EXECUTE_PERM: + case READ_WRITE_PERM: + case ALL_PERM: + ret = WRITE_ACCCESS_TYPE; + break; + default: + break; + } + return ret.toLowerCase(); + } + + private String getAccessTypesAsString(RangerAccessRequest request) { + String ret = null; + Map<String,Object> context = request.getContext(); + Set<String> accessTypes = null; + + Object val = context.get(RangerAccessRequestUtil.KEY_CONTEXT_ACCESSTYPES); + if (val instanceof Set<?>) { + try { + accessTypes = (Set<String>) val; + ret = getFormattedAccessType(accessTypes); + } catch (Throwable t) { + LOG.error("getAccessTypesAsString(): failed to get accessTypes from context", t); + } + } + return ret; + } + + private String getFormattedAccessType(Set<String> accessTypes) { + String ret = null; + if (CollectionUtils.isNotEmpty(accessTypes)) { + ret = String.join(", ", accessTypes); + } + return ret; + } + + private void addOperationNameToAuditEvent(){ + if (StringUtils.isNotBlank(operationName)) { + String opName = operationName; + if (StringUtils.isNotBlank(callerContext)) { + opName = opName + "/" + callerContext; + } + auditEvent.setAccessType(operationName); + auditEvent.setRequestData(opName); + } + } } diff --git a/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/RangerHdfsAuthorizerTest.java b/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/RangerHdfsAuthorizerTest.java index cf9bad5..e96ea5c 100644 --- a/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/RangerHdfsAuthorizerTest.java +++ b/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/RangerHdfsAuthorizerTest.java @@ -30,6 +30,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.server.namenode.INode; +import org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider; import org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer; import org.apache.hadoop.hdfs.server.namenode.INodeAttributes; import org.apache.hadoop.security.AccessControlException; @@ -91,11 +92,31 @@ public class RangerHdfsAuthorizerTest { */ public void checkDirAccess(FsAction access, String userName, String... groups) throws AccessControlException { final UserGroupInformation user = UserGroupInformation.createUserForTesting(userName, groups); - rangerControlEnforcer.checkPermission(FILE_OWNER, FILE_GROUP, user, - Arrays.copyOf(attributes, attributes.length - 1), Arrays.copyOf(nodes, nodes.length - 1), - new byte[0][0], SNAPSHOT_ID, path, ancestorIndex - 1, false /* doCheckOwner */, - null /* ancestorAccess */, null /* parentAccess */ , access, null /* subAccess */ , - false /* ignoreEmptyDir */); + + INodeAttributeProvider.AuthorizationContext.Builder builder = + new INodeAttributeProvider.AuthorizationContext.Builder() + .fsOwner(FILE_OWNER) + .supergroup(FILE_GROUP) + .callerUgi(user) + .inodeAttrs(Arrays.copyOf(attributes, attributes.length - 1)) + .inodes(Arrays.copyOf(nodes, nodes.length - 1)) + .pathByNameArr(new byte[0][0]) + .snapshotId(SNAPSHOT_ID) + .path(path) + .ancestorIndex(ancestorIndex - 1) + .doCheckOwner(false) + .ancestorAccess(null) + .parentAccess(null) + .access(access) + .subAccess(null) + .ignoreEmptyDir(false) + .operationName(null) + .callerContext(null); + + INodeAttributeProvider.AuthorizationContext authorizationContext + = new INodeAttributeProvider.AuthorizationContext(builder); + + rangerControlEnforcer.checkPermissionWithContext(authorizationContext); } /** @@ -104,9 +125,31 @@ public class RangerHdfsAuthorizerTest { */ public void checkAccess(FsAction access, String userName, String... groups) throws AccessControlException { final UserGroupInformation user = UserGroupInformation.createUserForTesting(userName, groups); - rangerControlEnforcer.checkPermission(FILE_OWNER, FILE_GROUP, user, attributes, nodes, new byte[0][0], - SNAPSHOT_ID, path, ancestorIndex, false /* doCheckOwner */, null /* ancestorAccess */, - null /* parentAccess */ , access, null /* subAccess */ , false /* ignoreEmptyDir */); + + INodeAttributeProvider.AuthorizationContext.Builder builder = + new INodeAttributeProvider.AuthorizationContext.Builder() + .fsOwner(FILE_OWNER) + .supergroup(FILE_GROUP) + .callerUgi(user) + .inodeAttrs(attributes) + .inodes(nodes) + .pathByNameArr(new byte[0][0]) + .snapshotId(SNAPSHOT_ID) + .path(path) + .ancestorIndex(ancestorIndex - 1) + .doCheckOwner(false) + .ancestorAccess(null) + .parentAccess(null) + .access(access) + .subAccess(null) + .ignoreEmptyDir(false) + .operationName(null) + .callerContext(null); + + INodeAttributeProvider.AuthorizationContext authorizationContext + = new INodeAttributeProvider.AuthorizationContext(builder); + + rangerControlEnforcer.checkPermissionWithContext(authorizationContext); } /** diff --git a/security-admin/src/main/webapp/scripts/utils/XAViewUtils.js b/security-admin/src/main/webapp/scripts/utils/XAViewUtils.js index 8b67452..e9edc9a 100644 --- a/security-admin/src/main/webapp/scripts/utils/XAViewUtils.js +++ b/security-admin/src/main/webapp/scripts/utils/XAViewUtils.js @@ -31,7 +31,7 @@ define(function(require) { XAViewUtil.resourceTypeFormatter = function(rawValue, model){ var resourcePath = _.isUndefined(model.get('resourcePath')) ? undefined : model.get('resourcePath'); var resourceType = _.isUndefined(model.get('resourceType')) ? undefined : model.get('resourceType'); - if((model.get('serviceType') === XAEnums.ServiceType.Service_HIVE.label || model.get('serviceType') === XAEnums.ServiceType.Service_HBASE.label || model.get('serviceType') === XAEnums.ServiceType.Service_SOLR.label) + if((model.get('serviceType') === XAEnums.ServiceType.Service_HIVE.label || model.get('serviceType') === XAEnums.ServiceType.Service_HBASE.label || model.get('serviceType') === XAEnums.ServiceType.Service_SOLR.label || model.get('serviceType') === XAEnums.ServiceType.Service_HDFS.label) && model.get('aclEnforcer') === "ranger-acl" && model.get('requestData')){ if(resourcePath && !_.isEmpty(model.get('requestData'))) { @@ -65,13 +65,14 @@ define(function(require) { }; XAViewUtil.showQueryPopup = function(model, that){ - if((model.get('serviceType') === XAEnums.ServiceType.Service_HIVE.label || model.get('serviceType') === XAEnums.ServiceType.Service_HBASE.label || model.get('serviceType') === XAEnums.ServiceType.Service_SOLR.label) + if((model.get('serviceType') === XAEnums.ServiceType.Service_HIVE.label || model.get('serviceType') === XAEnums.ServiceType.Service_HBASE.label || model.get('serviceType') === XAEnums.ServiceType.Service_SOLR.label || model.get('serviceType') === XAEnums.ServiceType.Service_HDFS.label) && model.get('aclEnforcer') === "ranger-acl" && model.get('requestData') && !_.isEmpty(model.get('requestData'))){ var titleMap = {}; titleMap[XAEnums.ServiceType.Service_HIVE.label] = 'Hive Query'; titleMap[XAEnums.ServiceType.Service_HBASE.label] = 'HBase Audit Data'; titleMap[XAEnums.ServiceType.Service_SOLR.label] = 'Solr Query'; + titleMap[XAEnums.ServiceType.Service_HDFS.label] = 'HDFS Operation Name'; var msg = '<div class="pull-right link-tag query-icon copyQuery btn btn-sm" title="Copy Query"><i class="fa-fw fa fa-copy"></i></div><div class="query-content">'+model.get('requestData')+'</div>'; var $elements = that.$el.find('table [data-name = "queryInfo"][data-id = "'+model.id+'"]'); $elements.popover({
