Author: eli
Date: Tue Jun 26 18:18:19 2012
New Revision: 1354147
URL: http://svn.apache.org/viewvc?rev=1354147&view=rev
Log:
HDFS-3535. Audit logging should log denied accesses. Contributed by Andy
Isaacson
Added:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
- copied unchanged from r1354144,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project:r1354144
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1354144
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1354147&r1=1354146&r2=1354147&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Tue Jun 26 18:18:19 2012
@@ -82,6 +82,8 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3516. Check content-type in WebHdfsFileSystem. (szetszwo)
+ HDFS-3535. Audit logging should log denied accesses. (Andy Isaacson via
eli)
+
OPTIMIZATIONS
HDFS-2982. Startup performance suffers when there are many edit log
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1354144
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1354147&r1=1354146&r2=1354147&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Tue Jun 26 18:18:19 2012
@@ -225,8 +225,15 @@ public class FSNamesystem implements Nam
private static final void logAuditEvent(UserGroupInformation ugi,
InetAddress addr, String cmd, String src, String dst,
HdfsFileStatus stat) {
+ logAuditEvent(true, ugi, addr, cmd, src, dst, stat);
+ }
+
+ private static final void logAuditEvent(boolean succeeded,
+ UserGroupInformation ugi, InetAddress addr, String cmd, String src,
+ String dst, HdfsFileStatus stat) {
final StringBuilder sb = auditBuffer.get();
sb.setLength(0);
+ sb.append("allowed=").append(succeeded).append("\t");
sb.append("ugi=").append(ugi).append("\t");
sb.append("ip=").append(addr).append("\t");
sb.append("cmd=").append(cmd).append("\t");
@@ -1003,6 +1010,21 @@ public class FSNamesystem implements Nam
void setPermission(String src, FsPermission permission)
throws AccessControlException, FileNotFoundException, SafeModeException,
UnresolvedLinkException, IOException {
+ try {
+ setPermissionInt(src, permission);
+ } catch (AccessControlException e) {
+ if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+ logAuditEvent(false, UserGroupInformation.getCurrentUser(),
+ Server.getRemoteIp(),
+ "setPermission", src, null, null);
+ }
+ throw e;
+ }
+ }
+
+ private void setPermissionInt(String src, FsPermission permission)
+ throws AccessControlException, FileNotFoundException, SafeModeException,
+ UnresolvedLinkException, IOException {
HdfsFileStatus resultingStat = null;
writeLock();
try {
@@ -1034,6 +1056,21 @@ public class FSNamesystem implements Nam
void setOwner(String src, String username, String group)
throws AccessControlException, FileNotFoundException, SafeModeException,
UnresolvedLinkException, IOException {
+ try {
+ setOwnerInt(src, username, group);
+ } catch (AccessControlException e) {
+ if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+ logAuditEvent(false, UserGroupInformation.getCurrentUser(),
+ Server.getRemoteIp(),
+ "setOwner", src, null, null);
+ }
+ throw e;
+ }
+ }
+
+ private void setOwnerInt(String src, String username, String group)
+ throws AccessControlException, FileNotFoundException, SafeModeException,
+ UnresolvedLinkException, IOException {
HdfsFileStatus resultingStat = null;
writeLock();
try {
@@ -1091,6 +1128,22 @@ public class FSNamesystem implements Nam
LocatedBlocks getBlockLocations(String src, long offset, long length,
boolean doAccessTime, boolean needBlockToken, boolean checkSafeMode)
throws FileNotFoundException, UnresolvedLinkException, IOException {
+ try {
+ return getBlockLocationsInt(src, offset, length, doAccessTime,
+ needBlockToken, checkSafeMode);
+ } catch (AccessControlException e) {
+ if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+ logAuditEvent(false, UserGroupInformation.getCurrentUser(),
+ Server.getRemoteIp(),
+ "open", src, null, null);
+ }
+ throw e;
+ }
+ }
+
+ private LocatedBlocks getBlockLocationsInt(String src, long offset, long
length,
+ boolean doAccessTime, boolean needBlockToken, boolean checkSafeMode)
+ throws FileNotFoundException, UnresolvedLinkException, IOException {
if (isPermissionEnabled) {
checkPathAccess(src, FsAction.READ);
}
@@ -1187,6 +1240,20 @@ public class FSNamesystem implements Nam
*/
void concat(String target, String [] srcs)
throws IOException, UnresolvedLinkException {
+ try {
+ concatInt(target, srcs);
+ } catch (AccessControlException e) {
+ if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+ logAuditEvent(false, UserGroupInformation.getLoginUser(),
+ Server.getRemoteIp(),
+ "concat", Arrays.toString(srcs), target, null);
+ }
+ throw e;
+ }
+ }
+
+ private void concatInt(String target, String [] srcs)
+ throws IOException, UnresolvedLinkException {
if(FSNamesystem.LOG.isDebugEnabled()) {
FSNamesystem.LOG.debug("concat " + Arrays.toString(srcs) +
" to " + target);
@@ -1339,6 +1406,20 @@ public class FSNamesystem implements Nam
* written to the edits log but is not flushed.
*/
void setTimes(String src, long mtime, long atime)
+ throws IOException, UnresolvedLinkException {
+ try {
+ setTimesInt(src, mtime, atime);
+ } catch (AccessControlException e) {
+ if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+ logAuditEvent(false, UserGroupInformation.getCurrentUser(),
+ Server.getRemoteIp(),
+ "setTimes", src, null, null);
+ }
+ throw e;
+ }
+ }
+
+ private void setTimesInt(String src, long mtime, long atime)
throws IOException, UnresolvedLinkException {
if (!isAccessTimeSupported() && atime != -1) {
throw new IOException("Access time for hdfs is not configured. " +
@@ -1375,6 +1456,21 @@ public class FSNamesystem implements Nam
void createSymlink(String target, String link,
PermissionStatus dirPerms, boolean createParent)
throws IOException, UnresolvedLinkException {
+ try {
+ createSymlinkInt(target, link, dirPerms, createParent);
+ } catch (AccessControlException e) {
+ if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+ logAuditEvent(false, UserGroupInformation.getCurrentUser(),
+ Server.getRemoteIp(),
+ "createSymlink", link, target, null);
+ }
+ throw e;
+ }
+ }
+
+ private void createSymlinkInt(String target, String link,
+ PermissionStatus dirPerms, boolean createParent)
+ throws IOException, UnresolvedLinkException {
HdfsFileStatus resultingStat = null;
writeLock();
try {
@@ -1442,8 +1538,22 @@ public class FSNamesystem implements Nam
* @return true if successful;
* false if file does not exist or is a directory
*/
- boolean setReplication(final String src, final short replication
- ) throws IOException {
+ boolean setReplication(final String src, final short replication)
+ throws IOException {
+ try {
+ return setReplicationInt(src, replication);
+ } catch (AccessControlException e) {
+ if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+ logAuditEvent(false, UserGroupInformation.getCurrentUser(),
+ Server.getRemoteIp(),
+ "setReplication", src, null, null);
+ }
+ throw e;
+ }
+ }
+
+ private boolean setReplicationInt(final String src, final short replication)
+ throws IOException {
blockManager.verifyReplication(src, replication, null);
final boolean isFile;
@@ -1476,7 +1586,7 @@ public class FSNamesystem implements Nam
}
return isFile;
}
-
+
long getPreferredBlockSize(String filename)
throws IOException, UnresolvedLinkException {
readLock();
@@ -1522,6 +1632,24 @@ public class FSNamesystem implements Nam
short replication, long blockSize) throws AccessControlException,
SafeModeException, FileAlreadyExistsException, UnresolvedLinkException,
FileNotFoundException, ParentNotDirectoryException, IOException {
+ try {
+ startFileInt(src, permissions, holder, clientMachine, flag, createParent,
+ replication, blockSize);
+ } catch (AccessControlException e) {
+ if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+ logAuditEvent(false, UserGroupInformation.getCurrentUser(),
+ Server.getRemoteIp(),
+ "create", src, null, null);
+ }
+ throw e;
+ }
+ }
+
+ private void startFileInt(String src, PermissionStatus permissions, String
holder,
+ String clientMachine, EnumSet<CreateFlag> flag, boolean createParent,
+ short replication, long blockSize) throws AccessControlException,
+ SafeModeException, FileAlreadyExistsException, UnresolvedLinkException,
+ FileNotFoundException, ParentNotDirectoryException, IOException {
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -1825,6 +1953,22 @@ public class FSNamesystem implements Nam
throws AccessControlException, SafeModeException,
FileAlreadyExistsException, FileNotFoundException,
ParentNotDirectoryException, IOException {
+ try {
+ return appendFileInt(src, holder, clientMachine);
+ } catch (AccessControlException e) {
+ if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+ logAuditEvent(false, UserGroupInformation.getCurrentUser(),
+ Server.getRemoteIp(),
+ "append", src, null, null);
+ }
+ throw e;
+ }
+ }
+
+ private LocatedBlock appendFileInt(String src, String holder, String
clientMachine)
+ throws AccessControlException, SafeModeException,
+ FileAlreadyExistsException, FileNotFoundException,
+ ParentNotDirectoryException, IOException {
if (!supportAppends) {
throw new UnsupportedOperationException(
"Append is not enabled on this NameNode. Use the " +
@@ -2311,6 +2455,20 @@ public class FSNamesystem implements Nam
*/
@Deprecated
boolean renameTo(String src, String dst)
+ throws IOException, UnresolvedLinkException {
+ try {
+ return renameToInt(src, dst);
+ } catch (AccessControlException e) {
+ if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+ logAuditEvent(false, UserGroupInformation.getCurrentUser(),
+ Server.getRemoteIp(),
+ "rename", src, dst, null);
+ }
+ throw e;
+ }
+ }
+
+ private boolean renameToInt(String src, String dst)
throws IOException, UnresolvedLinkException {
boolean status = false;
HdfsFileStatus resultingStat = null;
@@ -2422,20 +2580,35 @@ public class FSNamesystem implements Nam
* @see ClientProtocol#delete(String, boolean) for detailed descriptoin and
* description of exceptions
*/
- boolean delete(String src, boolean recursive)
- throws AccessControlException, SafeModeException,
- UnresolvedLinkException, IOException {
- if (NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug("DIR* NameSystem.delete: " + src);
- }
- boolean status = deleteInternal(src, recursive, true);
- if (status && auditLog.isInfoEnabled() && isExternalInvocation()) {
- logAuditEvent(UserGroupInformation.getCurrentUser(),
+ boolean delete(String src, boolean recursive)
+ throws AccessControlException, SafeModeException,
+ UnresolvedLinkException, IOException {
+ try {
+ return deleteInt(src, recursive);
+ } catch (AccessControlException e) {
+ if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+ logAuditEvent(false, UserGroupInformation.getCurrentUser(),
Server.getRemoteIp(),
"delete", src, null, null);
}
- return status;
+ throw e;
+ }
+ }
+
+ private boolean deleteInt(String src, boolean recursive)
+ throws AccessControlException, SafeModeException,
+ UnresolvedLinkException, IOException {
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* NameSystem.delete: " + src);
}
+ boolean status = deleteInternal(src, recursive, true);
+ if (status && auditLog.isInfoEnabled() && isExternalInvocation()) {
+ logAuditEvent(UserGroupInformation.getCurrentUser(),
+ Server.getRemoteIp(),
+ "delete", src, null, null);
+ }
+ return status;
+ }
/**
* Remove a file/directory from the namespace.
@@ -2591,6 +2764,20 @@ public class FSNamesystem implements Nam
*/
boolean mkdirs(String src, PermissionStatus permissions,
boolean createParent) throws IOException, UnresolvedLinkException {
+ try {
+ return mkdirsInt(src, permissions, createParent);
+ } catch (AccessControlException e) {
+ if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+ logAuditEvent(false, UserGroupInformation.getCurrentUser(),
+ Server.getRemoteIp(),
+ "mkdirs", src, null, null);
+ }
+ throw e;
+ }
+ }
+
+ private boolean mkdirsInt(String src, PermissionStatus permissions,
+ boolean createParent) throws IOException, UnresolvedLinkException {
boolean status = false;
if(NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* NameSystem.mkdirs: " + src);
@@ -3042,6 +3229,21 @@ public class FSNamesystem implements Nam
*/
DirectoryListing getListing(String src, byte[] startAfter,
boolean needLocation)
+ throws AccessControlException, UnresolvedLinkException, IOException {
+ try {
+ return getListingInt(src, startAfter, needLocation);
+ } catch (AccessControlException e) {
+ if (auditLog.isInfoEnabled() && isExternalInvocation()) {
+ logAuditEvent(false, UserGroupInformation.getCurrentUser(),
+ Server.getRemoteIp(),
+ "listStatus", src, null, null);
+ }
+ throw e;
+ }
+ }
+
+ private DirectoryListing getListingInt(String src, byte[] startAfter,
+ boolean needLocation)
throws AccessControlException, UnresolvedLinkException, IOException {
DirectoryListing dl;
readLock();
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1354144
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1354144
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1354144
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1354144
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1354144
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java?rev=1354147&r1=1354146&r2=1354147&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Tue Jun 26 18:18:19 2012
@@ -76,8 +76,9 @@ public class TestFsck {
"build/test") + "/audit.log";
// Pattern for:
- // ugi=name ip=/address cmd=FSCK src=/ dst=null perm=null
+ // allowed=true ugi=name ip=/address cmd=FSCK src=/ dst=null perm=null
static final Pattern fsckPattern = Pattern.compile(
+ "allowed=.*?\\s" +
"ugi=.*?\\s" +
"ip=/\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\s" +
"cmd=fsck\\ssrc=\\/\\sdst=null\\s" +
