HADOOP-12670 Fix TestNetUtils and TestSecurityUtil when localhost is ipv6 only
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fabf9c0c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fabf9c0c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fabf9c0c Branch: refs/heads/HADOOP-11890 Commit: fabf9c0cdc4ccd67cc0da92deb1c0c6b9bc4558f Parents: 7b681a6 Author: Elliott Clark <[email protected]> Authored: Tue Dec 22 15:38:40 2015 -0800 Committer: Elliott Clark <[email protected]> Committed: Tue Dec 29 13:53:49 2015 -0800 ---------------------------------------------------------------------- .../java/org/apache/hadoop/net/NetUtils.java | 2 +- .../apache/hadoop/security/SecurityUtil.java | 9 ++---- .../authorize/DefaultImpersonationProvider.java | 4 +-- .../org/apache/hadoop/net/TestNetUtils.java | 33 +++++++++++++++----- .../hadoop/security/TestDoAsEffectiveUser.java | 4 +-- .../hadoop/security/TestSecurityUtil.java | 26 ++++++++++----- 6 files changed, 51 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fabf9c0c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java index efb772d..f5cb387 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java @@ -650,7 +650,7 @@ public class NetUtils { if (InetAddressUtils.isIPv6Address(hostName)) { return "[" + hostName + "]:" + addr.getPort(); } - return hostName + ":" + addr.getPort(); + return hostName.toLowerCase() + ":" + addr.getPort(); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/fabf9c0c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java index 38096ab..86851bc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java @@ -386,7 +386,7 @@ public class SecurityUtil { if (token != null) { token.setService(service); if (LOG.isDebugEnabled()) { - LOG.debug("Acquired token "+token); // Token#toString() prints service + LOG.debug("Acquired token " + token); // Token#toString() prints service } } else { LOG.warn("Failed to get token for service "+service); @@ -400,18 +400,15 @@ public class SecurityUtil { * hadoop.security.token.service.use_ip */ public static Text buildTokenService(InetSocketAddress addr) { - String host = null; if (useIpForTokenService) { if (addr.isUnresolved()) { // host has no ip address throw new IllegalArgumentException( new UnknownHostException(addr.getHostName()) ); } - host = addr.getAddress().getHostAddress(); - } else { - host = StringUtils.toLowerCase(addr.getHostName()); + return new Text(NetUtils.getIPPortString(addr)); } - return new Text(host + ":" + addr.getPort()); + return new Text(NetUtils.getHostPortString(addr)); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/fabf9c0c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java index 26cd7ab..88ab5fe 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java @@ -124,10 +124,10 @@ public class DefaultImpersonationProvider implements ImpersonationProvider { + " is not allowed to impersonate " + user.getUserName()); } - MachineList MachineList = proxyHosts.get( + MachineList machineList = proxyHosts.get( getProxySuperuserIpConfKey(realUser.getShortUserName())); - if(MachineList == null || !MachineList.includes(remoteAddress)) { + if(machineList == null || !machineList.includes(remoteAddress)) { throw new AuthorizationException("Unauthorized connection for super-user: " + realUser.getUserName() + " from IP " + remoteAddress); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/fabf9c0c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java index ddb1f83..cc9666a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java @@ -489,13 +489,20 @@ public class TestNetUtils { return addr; } + private void - verifyInetAddress(InetAddress addr, String host, String ip) { + verifyInetAddress(InetAddress addr, String host, String... ips) { assertNotNull(addr); assertEquals(host, addr.getHostName()); - assertEquals(ip, addr.getHostAddress()); + + boolean found = false; + for (String ip:ips) { + found |= ip.equals(addr.getHostAddress()); + } + assertTrue("Expected addr.getHostAddress["+addr.getHostAddress()+"] to be one of " + StringUtils.join(ips, ","), found); } - + + @Test public void testResolverUnqualified() { String host = "host"; @@ -525,12 +532,16 @@ public class TestNetUtils { } // localhost - + @Test public void testResolverLoopback() { String host = "Localhost"; InetAddress addr = verifyResolve(host); // no lookup should occur - verifyInetAddress(addr, "Localhost", "127.0.0.1"); + verifyInetAddress(addr, + "Localhost", + "127.0.0.1", + IPV6_LOOPBACK_LONG_STRING, + IPV6_LOOPBACK_SHORT_STRING); } @Test @@ -637,10 +648,14 @@ public class TestNetUtils { // when ipaddress is normalized, same address is expected in return assertEquals(summary, hosts.get(0), normalizedHosts.get(0)); // for normalizing a resolvable hostname, resolved ipaddress is expected in return + assertFalse("Element 1 equal "+ summary, normalizedHosts.get(1).equals(hosts.get(1))); - assertEquals(summary, hosts.get(0), normalizedHosts.get(1)); - // this address HADOOP-8372: when normalizing a valid resolvable hostname start with numeric, + assertTrue("Should get the localhost address back", + normalizedHosts.get(1).equals(hosts.get(0)) || + normalizedHosts.get(1).equals(IPV6_LOOPBACK_LONG_STRING)); + // this address HADOOP-8372: when normalizing a valid resolvable hostname start with numeric, + // its ipaddress is expected to return assertFalse("Element 2 equal " + summary, normalizedHosts.get(2).equals(hosts.get(2))); @@ -690,7 +705,9 @@ public class TestNetUtils { InetSocketAddress addr = NetUtils.createSocketAddr(defaultAddr); conf.setSocketAddr("myAddress", addr); - assertEquals(defaultAddr.trim(), NetUtils.getHostPortString(addr)); + assertTrue( + "Trim should have been called on ipv6 hostname", + defaultAddr.trim().equalsIgnoreCase(NetUtils.getHostPortString(addr))); } private <T> void assertBetterArrayEquals(T[] expect, T[]got) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/fabf9c0c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java index b44fa8b..075764b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java @@ -431,7 +431,7 @@ public class TestDoAsEffectiveUser { public void testProxyWithToken() throws Exception { final Configuration conf = new Configuration(masterConf); TestTokenSecretManager sm = new TestTokenSecretManager(); - SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf); + UserGroupInformation.setConfiguration(conf); final Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class) .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0) @@ -485,7 +485,7 @@ public class TestDoAsEffectiveUser { public void testTokenBySuperUser() throws Exception { TestTokenSecretManager sm = new TestTokenSecretManager(); final Configuration newConf = new Configuration(masterConf); - SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, newConf); + UserGroupInformation.setConfiguration(newConf); final Server server = new RPC.Builder(newConf) .setProtocol(TestProtocol.class).setInstance(new TestImpl()) http://git-wip-us.apache.org/repos/asf/hadoop/blob/fabf9c0c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java index 14f9091..8b37bea 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java @@ -163,15 +163,15 @@ public class TestSecurityUtil { @Test public void testBuildTokenServiceSockAddr() { SecurityUtil.setTokenServiceUseIp(true); - assertEquals("127.0.0.1:123", - SecurityUtil.buildTokenService(new InetSocketAddress("LocalHost", 123)).toString() - ); - assertEquals("127.0.0.1:123", - SecurityUtil.buildTokenService(new InetSocketAddress("127.0.0.1", 123)).toString() + assertOneOf( + SecurityUtil.buildTokenService(NetUtils.createSocketAddrForHost("LocalHost", 123)).toString(), + "127.0.0.1:123", + "[0:0:0:0:0:0:0:1]:123" ); - // what goes in, comes out - assertEquals("127.0.0.1:123", - SecurityUtil.buildTokenService(NetUtils.createSocketAddr("127.0.0.1", 123)).toString() + assertOneOf( + SecurityUtil.buildTokenService(NetUtils.createSocketAddrForHost("127.0.0.1", 123)).toString(), + "127.0.0.1:123", + "[0:0:0:0:0:0:0:1]:123" ); } @@ -394,4 +394,14 @@ public class TestSecurityUtil { SecurityUtil.setAuthenticationMethod(KERBEROS, conf); assertEquals("kerberos", conf.get(HADOOP_SECURITY_AUTHENTICATION)); } + + private void assertOneOf(String value, String... expected) { + boolean found = false; + for (String ip : expected) { + found |= ip.equals(value); + } + assertTrue("Expected value [" + value + "] to be one of " + + org.apache.commons.lang.StringUtils.join(expected, ","), found); + } + }
