Author: wang
Date: Tue Dec 10 02:36:54 2013
New Revision: 1549746
URL: http://svn.apache.org/r1549746
Log:
Backport HDFS-4983 to branch-2.3. Numeric usernames do not work with WebHDFS
FS. Contributed by Yongjun Zhang.
Modified:
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
Modified:
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1549746&r1=1549745&r2=1549746&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Tue Dec 10 02:36:54 2013
@@ -25,6 +25,9 @@ Release 2.3.0 - UNRELEASED
HDFS-5568. Support includeSnapshots option with Fsck command. (Vinayakumar
B via umamahesh)
+ HDFS-4983. Numeric usernames do not work with WebHDFS FS. (Yongjun Zhang
via
+ jing9)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1549746&r1=1549745&r2=1549746&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
(original)
+++
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
Tue Dec 10 02:36:54 2013
@@ -156,6 +156,8 @@ public class DFSConfigKeys extends Commo
public static final int
DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_DEFAULT = 4;
public static final String DFS_WEBHDFS_ENABLED_KEY = "dfs.webhdfs.enabled";
public static final boolean DFS_WEBHDFS_ENABLED_DEFAULT = false;
+ public static final String DFS_WEBHDFS_USER_PATTERN_KEY =
"dfs.webhdfs.user.provider.user.pattern";
+ public static final String DFS_WEBHDFS_USER_PATTERN_DEFAULT =
"^[A-Za-z_][A-Za-z0-9._-]*[$]?$";
public static final String DFS_PERMISSIONS_ENABLED_KEY =
"dfs.permissions.enabled";
public static final boolean DFS_PERMISSIONS_ENABLED_DEFAULT = true;
public static final String DFS_PERMISSIONS_SUPERUSERGROUP_KEY =
"dfs.permissions.superusergroup";
Modified:
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java?rev=1549746&r1=1549745&r2=1549746&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
(original)
+++
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java
Tue Dec 10 02:36:54 2013
@@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.web.AuthFilter;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.hdfs.web.resources.Param;
+import org.apache.hadoop.hdfs.web.resources.UserParam;
import org.apache.hadoop.http.HttpServer;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
@@ -84,7 +85,10 @@ public class NameNodeHttpServer {
DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
}
if (WebHdfsFileSystem.isEnabled(conf, LOG)) {
- //add SPNEGO authentication filter for webhdfs
+ // set user pattern based on configuration file
+
UserParam.setUserPattern(conf.get(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
+
+ // add SPNEGO authentication filter for webhdfs
final String name = "SPNEGO";
final String classname = AuthFilter.class.getName();
final String pathSpec = WebHdfsFileSystem.PATH_PREFIX + "/*";
Modified:
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1549746&r1=1549745&r2=1549746&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
(original)
+++
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
Tue Dec 10 02:36:54 2013
@@ -182,6 +182,9 @@ public class WebHdfsFileSystem extends F
) throws IOException {
super.initialize(uri, conf);
setConf(conf);
+ /** set user pattern based on configuration file */
+
UserParam.setUserPattern(conf.get(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
+
ugi = UserGroupInformation.getCurrentUser();
try {
this.uri = new URI(uri.getScheme(), uri.getAuthority(), null, null,
null);
Modified:
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java?rev=1549746&r1=1549745&r2=1549746&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java
(original)
+++
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java
Tue Dec 10 02:36:54 2013
@@ -17,8 +17,10 @@
*/
package org.apache.hadoop.hdfs.web.resources;
+import static
org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT;
import org.apache.hadoop.security.UserGroupInformation;
-
+import com.google.common.annotations.VisibleForTesting;
+
import java.text.MessageFormat;
import java.util.regex.Pattern;
@@ -29,8 +31,21 @@ public class UserParam extends StringPar
/** Default parameter value. */
public static final String DEFAULT = "";
- private static final Domain DOMAIN = new Domain(NAME,
- Pattern.compile("^[A-Za-z_][A-Za-z0-9._-]*[$]?$"));
+ private static Domain domain = new Domain(NAME,
Pattern.compile(DFS_WEBHDFS_USER_PATTERN_DEFAULT));
+
+ @VisibleForTesting
+ public static Domain getUserPatternDomain() {
+ return domain;
+ }
+
+ @VisibleForTesting
+ public static void setUserPatternDomain(Domain dm) {
+ domain = dm;
+ }
+
+ public static void setUserPattern(String pattern) {
+ domain = new Domain(NAME, Pattern.compile(pattern));
+ }
private static String validateLength(String str) {
if (str == null) {
@@ -50,7 +65,7 @@ public class UserParam extends StringPar
* @param str a string representation of the parameter value.
*/
public UserParam(final String str) {
- super(DOMAIN, str == null || str.equals(DEFAULT)? null :
validateLength(str));
+ super(domain, str == null || str.equals(DEFAULT)? null :
validateLength(str));
}
/**
@@ -64,4 +79,4 @@ public class UserParam extends StringPar
public String getName() {
return NAME;
}
-}
\ No newline at end of file
+}
Modified:
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1549746&r1=1549745&r2=1549746&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
(original)
+++
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
Tue Dec 10 02:36:54 2013
@@ -1441,4 +1441,12 @@
</description>
</property>
+<property>
+ <name>dfs.webhdfs.user.provider.user.pattern</name>
+ <value>^[A-Za-z_][A-Za-z0-9._-]*[$]?$</value>
+ <description>
+ Valid pattern for user and group names for webhdfs, it must be a valid
java regex.
+ </description>
+</property>
+
</configuration>
Modified:
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java?rev=1549746&r1=1549745&r2=1549746&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
(original)
+++
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
Tue Dec 10 02:36:54 2013
@@ -258,4 +258,30 @@ public class TestWebHDFS {
cluster.shutdown();
}
}
+
+ @Test(timeout=300000)
+ public void testNumericalUserName() throws Exception {
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ conf.set(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
"^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$");
+ final MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ try {
+ cluster.waitActive();
+ WebHdfsTestUtil.getWebHdfsFileSystem(conf).setPermission(new Path("/"),
+ new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
+
+ UserGroupInformation.createUserForTesting("123", new
String[]{"my-group"})
+ .doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws IOException, URISyntaxException {
+ FileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf);
+ Path d = new Path("/my-dir");
+ Assert.assertTrue(fs.mkdirs(d));
+ return null;
+ }
+ });
+ } finally {
+ cluster.shutdown();
+ }
+ }
}
Modified:
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java?rev=1549746&r1=1549745&r2=1549746&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
(original)
+++
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
Tue Dec 10 02:36:54 2013
@@ -285,4 +285,19 @@ public class TestParam {
Assert.assertEquals(expected, computed.getValue());
}
}
+
+ @Test
+ public void testUserNameOkAfterResettingPattern() {
+ UserParam.Domain oldDomain = UserParam.getUserPatternDomain();
+
+ String newPattern = "^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$";
+ UserParam.setUserPattern(newPattern);
+
+ UserParam userParam = new UserParam("1x");
+ assertNotNull(userParam.getValue());
+ userParam = new UserParam("123");
+ assertNotNull(userParam.getValue());
+
+ UserParam.setUserPatternDomain(oldDomain);
+ }
}
