Author: eli
Date: Mon Sep 10 19:07:10 2012
New Revision: 1383054
URL: http://svn.apache.org/viewvc?rev=1383054&view=rev
Log:
HDFS-3907. Allow multiple users for local block readers. Contributed by Eli
Collins
Modified:
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/ (props
changed)
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/
(props changed)
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
(props changed)
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
(props changed)
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
(props changed)
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
(props changed)
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
(props changed)
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java
Propchange: hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project:r1383031
Propchange:
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1383031
Modified:
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1383054&r1=1383053&r2=1383054&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Mon Sep 10 19:07:10 2012
@@ -250,6 +250,8 @@ Release 2.0.2-alpha - 2012-09-07
HDFS-3888. Clean up BlockPlacementPolicyDefault. (Jing Zhao via szetszwo)
+ HDFS-3907. Allow multiple users for local block readers. (eli)
+
OPTIMIZATIONS
HDFS-2982. Startup performance suffers when there are many edit log
Propchange:
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1383031
Modified:
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1383054&r1=1383053&r2=1383054&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
(original)
+++
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
Mon Sep 10 19:07:10 2012
@@ -281,7 +281,7 @@ public class DataNode extends Configured
private AbstractList<File> dataDirs;
private Configuration conf;
- private final String userWithLocalPathAccess;
+ private final List<String> usersWithLocalPathAccess;
private boolean connectToDnViaHostname;
ReadaheadPool readaheadPool;
private final boolean getHdfsBlockLocationsEnabled;
@@ -304,8 +304,8 @@ public class DataNode extends Configured
final SecureResources resources) throws IOException {
super(conf);
- this.userWithLocalPathAccess =
- conf.get(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY);
+ this.usersWithLocalPathAccess = Arrays.asList(
+
conf.getTrimmedStrings(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY));
this.connectToDnViaHostname = conf.getBoolean(
DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME,
DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT);
@@ -1012,7 +1012,7 @@ public class DataNode extends Configured
private void checkBlockLocalPathAccess() throws IOException {
checkKerberosAuthMethod("getBlockLocalPathInfo()");
String currentUser =
UserGroupInformation.getCurrentUser().getShortUserName();
- if (!currentUser.equals(this.userWithLocalPathAccess)) {
+ if (!usersWithLocalPathAccess.contains(currentUser)) {
throw new AccessControlException(
"Can't continue with getBlockLocalPathInfo() "
+ "authorization. The user " + currentUser
Propchange:
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1383031
Propchange:
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1383031
Propchange:
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1383031
Propchange:
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1383031
Propchange:
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1383031
Modified:
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java?rev=1383054&r1=1383053&r2=1383054&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java
(original)
+++
hadoop/common/branches/branch-2.0.2-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java
Mon Sep 10 19:07:10 2012
@@ -224,7 +224,8 @@ public class TestShortCircuitLocalRead {
@Test
public void testGetBlockLocalPathInfo() throws IOException,
InterruptedException {
final Configuration conf = new Configuration();
- conf.set(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY,
"alloweduser");
+ conf.set(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY,
+ "alloweduser1,alloweduser2");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
.format(true).build();
cluster.waitActive();
@@ -232,8 +233,10 @@ public class TestShortCircuitLocalRead {
FileSystem fs = cluster.getFileSystem();
try {
DFSTestUtil.createFile(fs, new Path("/tmp/x"), 16, (short) 1, 23);
- UserGroupInformation aUgi = UserGroupInformation
- .createRemoteUser("alloweduser");
+ UserGroupInformation aUgi1 =
+ UserGroupInformation.createRemoteUser("alloweduser1");
+ UserGroupInformation aUgi2 =
+ UserGroupInformation.createRemoteUser("alloweduser2");
LocatedBlocks lb = cluster.getNameNode().getRpcServer()
.getBlockLocations("/tmp/x", 0, 16);
// Create a new block object, because the block inside LocatedBlock at
@@ -241,7 +244,7 @@ public class TestShortCircuitLocalRead {
ExtendedBlock blk = new ExtendedBlock(lb.get(0).getBlock());
Token<BlockTokenIdentifier> token = lb.get(0).getBlockToken();
final DatanodeInfo dnInfo = lb.get(0).getLocations()[0];
- ClientDatanodeProtocol proxy = aUgi
+ ClientDatanodeProtocol proxy = aUgi1
.doAs(new PrivilegedExceptionAction<ClientDatanodeProtocol>() {
@Override
public ClientDatanodeProtocol run() throws Exception {
@@ -250,13 +253,29 @@ public class TestShortCircuitLocalRead {
}
});
- //This should succeed
+ // This should succeed
BlockLocalPathInfo blpi = proxy.getBlockLocalPathInfo(blk, token);
Assert.assertEquals(
DataNodeTestUtils.getFSDataset(dn).getBlockLocalPathInfo(blk).getBlockPath(),
blpi.getBlockPath());
- // Now try with a not allowed user.
+ // Try with the other allowed user
+ proxy = aUgi2
+ .doAs(new PrivilegedExceptionAction<ClientDatanodeProtocol>() {
+ @Override
+ public ClientDatanodeProtocol run() throws Exception {
+ return DFSUtil.createClientDatanodeProtocolProxy(dnInfo, conf,
+ 60000, false);
+ }
+ });
+
+ // This should succeed as well
+ blpi = proxy.getBlockLocalPathInfo(blk, token);
+ Assert.assertEquals(
+
DataNodeTestUtils.getFSDataset(dn).getBlockLocalPathInfo(blk).getBlockPath(),
+ blpi.getBlockPath());
+
+ // Now try with a disallowed user
UserGroupInformation bUgi = UserGroupInformation
.createRemoteUser("notalloweduser");
proxy = bUgi
