Author: arp
Date: Sun Sep 22 18:23:22 2013
New Revision: 1525410
URL: http://svn.apache.org/r1525410
Log:
Merging r1524587 through r1525408 from trunk to branch HDFS-2832.
Added:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/site/resources/images/federation-background.gif
- copied unchanged from r1525408,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/site/resources/images/federation-background.gif
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/site/resources/images/federation.gif
- copied unchanged from r1525408,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/site/resources/images/federation.gif
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestOutOfOrderWrite.java
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/Federation.apt.vm
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
(props changed)
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
Propchange: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1524587-1525408
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestOutOfOrderWrite.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestOutOfOrderWrite.java?rev=1525410&r1=1525409&r2=1525410&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestOutOfOrderWrite.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestOutOfOrderWrite.java
Sun Sep 22 18:23:22 2013
@@ -33,8 +33,8 @@ import org.apache.hadoop.nfs.nfs3.reques
import org.apache.hadoop.nfs.nfs3.request.WRITE3Request;
import org.apache.hadoop.oncrpc.RegistrationClient;
import org.apache.hadoop.oncrpc.RpcCall;
-import org.apache.hadoop.oncrpc.RpcFrameDecoder;
import org.apache.hadoop.oncrpc.RpcReply;
+import org.apache.hadoop.oncrpc.RpcUtil;
import org.apache.hadoop.oncrpc.SimpleTcpClient;
import org.apache.hadoop.oncrpc.SimpleTcpClientHandler;
import org.apache.hadoop.oncrpc.XDR;
@@ -136,8 +136,9 @@ public class TestOutOfOrderWrite {
protected ChannelPipelineFactory setPipelineFactory() {
this.pipelineFactory = new ChannelPipelineFactory() {
public ChannelPipeline getPipeline() {
- return Channels.pipeline(new RpcFrameDecoder(), new WriteHandler(
- request));
+ return Channels.pipeline(
+ RpcUtil.constructRpcFrameDecoder(),
+ new WriteHandler(request));
}
};
return this.pipelineFactory;
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1525410&r1=1525409&r2=1525410&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Sun Sep 22 18:23:22 2013
@@ -366,6 +366,9 @@ Release 2.1.1-beta - 2013-09-23
HDFS-5199 Add more debug trace for NFS READ and WRITE. (brandonli)
+ HDFS-5234 Move RpcFrameDecoder out of the public API.
+ (Haohui Mai via brandonli)
+
IMPROVEMENTS
HDFS-4513. Clarify in the WebHDFS REST API that all JSON respsonses may
@@ -479,6 +482,9 @@ Release 2.1.1-beta - 2013-09-23
HDFS-5219. Add configuration keys for retry policy in WebHDFSFileSystem.
(Haohui Mai via jing9)
+ HDFS-5231. Fix broken links in the document of HDFS Federation. (Haohui Mai
+ via jing9)
+
Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1524587-1525408
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1524587-1525408
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1524587-1525408
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1524587-1525408
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1524587-1525408
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/Federation.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/Federation.apt.vm?rev=1525410&r1=1525409&r2=1525410&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/Federation.apt.vm
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/Federation.apt.vm
Sun Sep 22 18:23:22 2013
@@ -28,7 +28,7 @@ HDFS Federation
* {Background}
-[./federation-background.gif] HDFS Layers
+[./images/federation-background.gif] HDFS Layers
HDFS has two main layers:
@@ -72,7 +72,7 @@ HDFS Federation
send periodic heartbeats and block reports and handles commands from the
Namenodes.
-[./federation.gif] HDFS Federation Architecture
+[./images/federation.gif] HDFS Federation Architecture
<<Block Pool>>
Propchange:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1524587-1525408
Modified:
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java?rev=1525410&r1=1525409&r2=1525410&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
Sun Sep 22 18:23:22 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs;
import static org.junit.Assert.*;
import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
@@ -27,10 +28,15 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.junit.*;
public class TestGlobPaths {
+ private static final UserGroupInformation unprivilegedUser =
+ UserGroupInformation.createRemoteUser("myuser");
+
static class RegexPathFilter implements PathFilter {
private final String regex;
@@ -47,17 +53,33 @@ public class TestGlobPaths {
static private MiniDFSCluster dfsCluster;
static private FileSystem fs;
+ static private FileSystem unprivilegedFs;
static private FileContext fc;
+ static private FileContext unprivilegedFc;
static final private int NUM_OF_PATHS = 4;
static private String USER_DIR;
private Path[] path = new Path[NUM_OF_PATHS];
@BeforeClass
public static void setUp() throws Exception {
- Configuration conf = new HdfsConfiguration();
+ final Configuration conf = new HdfsConfiguration();
dfsCluster = new MiniDFSCluster.Builder(conf).build();
fs = FileSystem.get(conf);
+ unprivilegedFs =
+ unprivilegedUser.doAs(new PrivilegedExceptionAction<FileSystem>() {
+ @Override
+ public FileSystem run() throws IOException {
+ return FileSystem.get(conf);
+ }
+ });
fc = FileContext.getFileContext(conf);
+ unprivilegedFc =
+ unprivilegedUser.doAs(new PrivilegedExceptionAction<FileContext>() {
+ @Override
+ public FileContext run() throws IOException {
+ return FileContext.getFileContext(conf);
+ }
+ });
USER_DIR = fs.getHomeDirectory().toUri().getPath().toString();
}
@@ -781,8 +803,8 @@ public class TestGlobPaths {
* A glob test that can be run on either FileContext or FileSystem.
*/
private static interface FSTestWrapperGlobTest {
- void run(FSTestWrapper wrap, FileSystem fs, FileContext fc)
- throws Exception;
+ void run(FSTestWrapper wrap, FSTestWrapper unprivilegedWrapper,
+ FileSystem fs, FileContext fc) throws Exception;
}
/**
@@ -791,7 +813,8 @@ public class TestGlobPaths {
private void testOnFileSystem(FSTestWrapperGlobTest test) throws Exception {
try {
fc.mkdir(new Path(USER_DIR), FsPermission.getDefault(), true);
- test.run(new FileSystemTestWrapper(fs), fs, null);
+ test.run(new FileSystemTestWrapper(fs),
+ new FileSystemTestWrapper(unprivilegedFs), fs, null);
} finally {
fc.delete(new Path(USER_DIR), true);
}
@@ -803,7 +826,8 @@ public class TestGlobPaths {
private void testOnFileContext(FSTestWrapperGlobTest test) throws Exception {
try {
fs.mkdirs(new Path(USER_DIR));
- test.run(new FileContextTestWrapper(fc), null, fc);
+ test.run(new FileContextTestWrapper(fc),
+ new FileContextTestWrapper(unprivilegedFc), null, fc);
} finally {
cleanupDFS();
}
@@ -834,8 +858,8 @@ public class TestGlobPaths {
* Test globbing through symlinks.
*/
private static class TestGlobWithSymlinks implements FSTestWrapperGlobTest {
- public void run(FSTestWrapper wrap, FileSystem fs, FileContext fc)
- throws Exception {
+ public void run(FSTestWrapper wrap, FSTestWrapper unprivilegedWrap,
+ FileSystem fs, FileContext fc) throws Exception {
// Test that globbing through a symlink to a directory yields a path
// containing that symlink.
wrap.mkdir(new Path(USER_DIR + "/alpha"), FsPermission.getDirDefault(),
@@ -886,8 +910,8 @@ public class TestGlobPaths {
*/
private static class TestGlobWithSymlinksToSymlinks implements
FSTestWrapperGlobTest {
- public void run(FSTestWrapper wrap, FileSystem fs, FileContext fc)
- throws Exception {
+ public void run(FSTestWrapper wrap, FSTestWrapper unprivilegedWrap,
+ FileSystem fs, FileContext fc) throws Exception {
// Test that globbing through a symlink to a symlink to a directory
// fully resolves
wrap.mkdir(new Path(USER_DIR + "/alpha"), FsPermission.getDirDefault(),
@@ -961,8 +985,8 @@ public class TestGlobPaths {
*/
private static class TestGlobSymlinksWithCustomPathFilter implements
FSTestWrapperGlobTest {
- public void run(FSTestWrapper wrap, FileSystem fs, FileContext fc)
- throws Exception {
+ public void run(FSTestWrapper wrap, FSTestWrapper unprivilegedWrap,
+ FileSystem fs, FileContext fc) throws Exception {
// Test that globbing through a symlink to a symlink to a directory
// fully resolves
wrap.mkdir(new Path(USER_DIR + "/alpha"), FsPermission.getDirDefault(),
@@ -1009,8 +1033,8 @@ public class TestGlobPaths {
* Test that globStatus fills in the scheme even when it is not provided.
*/
private static class TestGlobFillsInScheme implements FSTestWrapperGlobTest {
- public void run(FSTestWrapper wrap, FileSystem fs, FileContext fc)
- throws Exception {
+ public void run(FSTestWrapper wrap, FSTestWrapper unprivilegedWrap,
+ FileSystem fs, FileContext fc) throws Exception {
// Verify that the default scheme is hdfs, when we don't supply one.
wrap.mkdir(new Path(USER_DIR + "/alpha"), FsPermission.getDirDefault(),
false);
@@ -1052,8 +1076,8 @@ public class TestGlobPaths {
* Test that globStatus works with relative paths.
**/
private static class TestRelativePath implements FSTestWrapperGlobTest {
- public void run(FSTestWrapper wrap, FileSystem fs, FileContext fc)
- throws Exception {
+ public void run(FSTestWrapper wrap, FSTestWrapper unprivilegedWrap,
+ FileSystem fs, FileContext fc) throws Exception {
String[] files = new String[] { "a", "abc", "abc.p", "bacd" };
Path[] path = new Path[files.length];
@@ -1086,4 +1110,44 @@ public class TestGlobPaths {
public void testRelativePathOnFC() throws Exception {
testOnFileContext(new TestRelativePath());
}
+
+ /**
+ * Test that trying to glob through a directory we don't have permission
+ * to list fails with AccessControlException rather than succeeding or
+ * throwing any other exception.
+ **/
+ private static class TestGlobAccessDenied implements FSTestWrapperGlobTest {
+ public void run(FSTestWrapper wrap, FSTestWrapper unprivilegedWrap,
+ FileSystem fs, FileContext fc) throws Exception {
+ wrap.mkdir(new Path("/nopermission/val"),
+ new FsPermission((short)0777), true);
+ wrap.mkdir(new Path("/norestrictions/val"),
+ new FsPermission((short)0777), true);
+ wrap.setPermission(new Path("/nopermission"),
+ new FsPermission((short)0));
+ try {
+ unprivilegedWrap.globStatus(new Path("/no*/*"),
+ new AcceptAllPathFilter());
+ Assert.fail("expected to get an AccessControlException when " +
+ "globbing through a directory we don't have permissions " +
+ "to list.");
+ } catch (AccessControlException ioe) {
+ }
+
+ Assert.assertEquals("/norestrictions/val",
+ TestPath.mergeStatuses(unprivilegedWrap.globStatus(
+ new Path("/norestrictions/*"),
+ new AcceptAllPathFilter())));
+ }
+ }
+
+ @Test
+ public void testGlobAccessDeniedOnFS() throws Exception {
+ testOnFileSystem(new TestGlobAccessDenied());
+ }
+
+ @Test
+ public void testGlobAccessDeniedOnFC() throws Exception {
+ testOnFileContext(new TestGlobAccessDenied());
+ }
}
