This is an automated email from the ASF dual-hosted git repository.
brahma pushed a commit to branch HADOOP-17800
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/HADOOP-17800 by this push:
new c0c70e0 YARN-4283. Avoid unsafe split and append on fields that might
be IPv6 literals. Contributed by Nemanja Matkovic and Hemanth Boyina
c0c70e0 is described below
commit c0c70e08338b47caf2fd84fad88d1381fe92998c
Author: Brahma Reddy Battula <bra...@apache.org>
AuthorDate: Wed Aug 4 17:49:52 2021 +0530
YARN-4283. Avoid unsafe split and append on fields that might be IPv6
literals. Contributed by Nemanja Matkovic and Hemanth Boyina
---
.../org/apache/hadoop/yarn/api/records/NodeId.java | 16 +++--
.../apache/hadoop/yarn/util/ConverterUtils.java | 12 ++--
.../hadoop/yarn/webapp/util/WebAppUtils.java | 19 +++--
.../hadoop/yarn/conf/TestYarnConfiguration.java | 84 ++++++++++++----------
.../hadoop/yarn/util/TestConverterUtils.java | 16 ++++-
.../org/apache/hadoop/yarn/lib/TestZKClient.java | 23 +++---
.../containermanager/ContainerManagerImpl.java | 3 +-
.../server/resourcemanager/ResourceManager.java | 5 +-
.../hadoop/yarn/server/resourcemanager/MockNM.java | 7 +-
.../hadoop/yarn/server/webproxy/WebAppProxy.java | 7 +-
.../webproxy/amfilter/AmFilterInitializer.java | 6 +-
.../server/webproxy/TestWebAppProxyServlet.java | 4 +-
12 files changed, 119 insertions(+), 83 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java
index a0b87a7..9080905 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeId.java
@@ -23,6 +23,7 @@ import
org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.util.Records;
+import com.google.common.net.HostAndPort;
/**
* <p><code>NodeId</code> is the unique identifier for a node.</p>
@@ -116,17 +117,18 @@ public abstract class NodeId implements
Comparable<NodeId> {
@Public
@Stable
public static NodeId fromString(String nodeIdStr) {
- String[] parts = nodeIdStr.split(":");
- if (parts.length != 2) {
- throw new IllegalArgumentException("Invalid NodeId [" + nodeIdStr
- + "]. Expected host:port");
+ HostAndPort hp = HostAndPort.fromString(nodeIdStr);
+ if (!hp.hasPort()) {
+ throw new IllegalArgumentException(
+ "Invalid NodeId [" + nodeIdStr + "]. Expected host:port");
}
try {
- NodeId nodeId =
- NodeId.newInstance(parts[0].trim(), Integer.parseInt(parts[1]));
+ String hostPortStr = hp.toString();
+ String host = hostPortStr.substring(0, hostPortStr.lastIndexOf(":"));
+ NodeId nodeId = NodeId.newInstance(host, hp.getPort());
return nodeId;
} catch (NumberFormatException e) {
- throw new IllegalArgumentException("Invalid port: " + parts[1], e);
+ throw new IllegalArgumentException("Invalid port: " + hp.getPort(), e);
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
index 67bc2b7..4ef8510 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.factories.RecordFactory;
-
+import com.google.common.net.HostAndPort;
/**
* This class contains a set of utilities which help converting data structures
@@ -114,11 +114,11 @@ public class ConverterUtils {
@Private
@InterfaceStability.Unstable
- public static NodeId toNodeIdWithDefaultPort(String nodeIdStr) {
- if (nodeIdStr.indexOf(":") < 0) {
- return NodeId.fromString(nodeIdStr + ":0");
- }
- return NodeId.fromString(nodeIdStr);
+ public static NodeId toNodeIdWithDefaultPort(
+ String nodeIdStr) {
+ HostAndPort hp = HostAndPort.fromString(nodeIdStr);
+ hp = hp.withDefaultPort(0);
+ return toNodeId(hp.toString());
}
/*
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
index 5b57819..6c28197 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
+import com.google.common.net.HostAndPort;
import javax.servlet.http.HttpServletRequest;
@@ -64,15 +65,13 @@ public class WebAppUtils {
public static void setRMWebAppPort(Configuration conf, int port) {
String hostname = getRMWebAppURLWithoutScheme(conf);
- hostname =
- (hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":"))
- : hostname;
- setRMWebAppHostnameAndPort(conf, hostname, port);
+ HostAndPort hp = HostAndPort.fromString(hostname);
+ setRMWebAppHostnameAndPort(conf, hp.getHost(), port);
}
public static void setRMWebAppHostnameAndPort(Configuration conf,
String hostname, int port) {
- String resolvedAddress = hostname + ":" + port;
+ String resolvedAddress = HostAndPort.fromParts(hostname, port).toString();
if (YarnConfiguration.useHttps(conf)) {
conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, resolvedAddress);
} else {
@@ -82,12 +81,11 @@ public class WebAppUtils {
public static void setNMWebAppHostNameAndPort(Configuration conf,
String hostName, int port) {
+ String hostPortString = HostAndPort.fromParts(hostName, port).toString();
if (YarnConfiguration.useHttps(conf)) {
- conf.set(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS,
- hostName + ":" + port);
+ conf.set(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS, hostPortString);
} else {
- conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS,
- hostName + ":" + port);
+ conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, hostPortString);
}
}
@@ -325,7 +323,8 @@ public class WebAppUtils {
String host = conf.getTrimmed(hostProperty);
if (host != null && !host.isEmpty()) {
if (webAppURLWithoutScheme.contains(":")) {
- webAppURLWithoutScheme = host + ":" +
webAppURLWithoutScheme.split(":")[1];
+ String[] splits = webAppURLWithoutScheme.split(":");
+ webAppURLWithoutScheme = host + ":" + splits[splits.length - 1];
}
else {
throw new YarnRuntimeException("webAppURLWithoutScheme must include
port specification but doesn't: " +
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
index 212e09c..502414b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.conf;
+import com.google.common.net.HostAndPort;
+import org.apache.hadoop.net.NetUtils;
import org.junit.Assert;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@@ -74,7 +76,7 @@ public class TestYarnConfiguration {
conf.set(YarnConfiguration.RM_ADDRESS, "rmtesting:9999");
String rmWebUrl = WebAppUtils.getRMWebAppURLWithScheme(conf);
String[] parts = rmWebUrl.split(":");
- Assert.assertEquals("RM Web URL Port is incrrect", 24543,
+ Assert.assertEquals("RM Web URL Port is incorrect", 24543,
Integer.parseInt(parts[parts.length - 1]));
Assert.assertNotSame(
"RM Web Url not resolved correctly. Should not be rmtesting",
@@ -112,10 +114,9 @@ public class TestYarnConfiguration {
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
- assertEquals(
- new InetSocketAddress(
- YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
- YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+ assertEquals(new InetSocketAddress(
+
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
resourceTrackerAddress);
//with address
@@ -125,10 +126,8 @@ public class TestYarnConfiguration {
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
- assertEquals(
- new InetSocketAddress(
- "10.0.0.1",
- YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+ assertEquals(new InetSocketAddress("10.0.0.1",
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
resourceTrackerAddress);
//address and socket
@@ -139,9 +138,23 @@ public class TestYarnConfiguration {
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
assertEquals(
- new InetSocketAddress(
- "10.0.0.2",
- 5001),
+ new InetSocketAddress(
+ "10.0.0.2",
+ 5001),
+ resourceTrackerAddress);
+
+ // IPv6 address and socket
+ conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ "[2401:db00:20:a01e:face:0:5:0]:5001");
+ resourceTrackerAddress = conf.getSocketAddr(
+ YarnConfiguration.RM_BIND_HOST,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
+ assertEquals(
+ new InetSocketAddress(
+ "2401:db00:20:a01e:face:0:5:0",
+ 5001),
resourceTrackerAddress);
//bind host only
@@ -152,10 +165,8 @@ public class TestYarnConfiguration {
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
- assertEquals(
- new InetSocketAddress(
- "10.0.0.3",
- YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+ assertEquals(new InetSocketAddress("10.0.0.3",
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
resourceTrackerAddress);
//bind host and address no port
@@ -166,10 +177,8 @@ public class TestYarnConfiguration {
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
- assertEquals(
- new InetSocketAddress(
- "0.0.0.0",
- YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+ assertEquals(new InetSocketAddress("0.0.0.0",
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
resourceTrackerAddress);
//bind host and address with port
@@ -180,10 +189,7 @@ public class TestYarnConfiguration {
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
- assertEquals(
- new InetSocketAddress(
- "0.0.0.0",
- 5003),
+ assertEquals(new InetSocketAddress("0.0.0.0", 5003),
resourceTrackerAddress);
}
@@ -197,9 +203,8 @@ public class TestYarnConfiguration {
//no override, old behavior. Won't work on a host named "yo.yo.yo"
conf = new YarnConfiguration();
conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo");
- serverAddress = new InetSocketAddress(
- YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
-
Integer.parseInt(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1]));
+ serverAddress = newInetSocketAddressFromHostPort(
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS);
resourceTrackerConnectAddress = conf.updateConnectAddr(
YarnConfiguration.RM_BIND_HOST,
@@ -207,15 +212,15 @@ public class TestYarnConfiguration {
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
serverAddress);
-
assertFalse(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo"));
+ assertFalse(NetUtils.getSocketAddressString(resourceTrackerConnectAddress)
+ .startsWith("yo.yo.yo"));
//cause override with address
conf = new YarnConfiguration();
conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo");
conf.set(YarnConfiguration.RM_BIND_HOST, "0.0.0.0");
- serverAddress = new InetSocketAddress(
- YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
-
Integer.parseInt(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1]));
+ serverAddress = newInetSocketAddressFromHostPort(
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS);
resourceTrackerConnectAddress = conf.updateConnectAddr(
YarnConfiguration.RM_BIND_HOST,
@@ -223,7 +228,8 @@ public class TestYarnConfiguration {
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
serverAddress);
-
assertTrue(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo"));
+ assertTrue(NetUtils.getSocketAddressString(resourceTrackerConnectAddress)
+ .startsWith("yo.yo.yo"));
//tests updateConnectAddr won't add suffix to NM service address
configurations
conf = new YarnConfiguration();
@@ -232,9 +238,8 @@ public class TestYarnConfiguration {
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.set(YarnConfiguration.RM_HA_ID, "rm1");
- serverAddress = new InetSocketAddress(
- YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS.split(":")[0],
-
Integer.parseInt(YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS.split(":")[1]));
+ serverAddress = newInetSocketAddressFromHostPort(
+ YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS);
InetSocketAddress localizerAddress = conf.updateConnectAddr(
YarnConfiguration.NM_BIND_HOST,
@@ -242,8 +247,15 @@ public class TestYarnConfiguration {
YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
serverAddress);
- assertTrue(localizerAddress.toString().startsWith("yo.yo.yo"));
+ assertTrue(NetUtils.getSocketAddressString(localizerAddress)
+ .startsWith("yo.yo.yo"));
assertNull(conf.get(
HAUtil.addSuffix(YarnConfiguration.NM_LOCALIZER_ADDRESS, "rm1")));
}
+
+ private InetSocketAddress newInetSocketAddressFromHostPort(
+ String hostPort) {
+ HostAndPort hp = HostAndPort.fromString(hostPort);
+ return new InetSocketAddress(hp.getHost(), hp.getPort());
+ }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
index 5754221..0256de21 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.junit.Test;
public class TestConverterUtils {
-
+
@Test
public void testConvertUrlWithNoPort() throws URISyntaxException {
Path expectedPath = new Path("hdfs://foo.com");
@@ -92,14 +92,24 @@ public class TestConverterUtils {
@Test
public void testNodeIdWithDefaultPort() throws URISyntaxException {
NodeId nid;
-
+
nid = ConverterUtils.toNodeIdWithDefaultPort("node:10");
assertThat(nid.getPort()).isEqualTo(10);
assertThat(nid.getHost()).isEqualTo("node");
-
+
nid = ConverterUtils.toNodeIdWithDefaultPort("node");
assertThat(nid.getPort()).isEqualTo(0);
assertThat(nid.getHost()).isEqualTo("node");
+
+ nid = ConverterUtils
+ .toNodeIdWithDefaultPort("[2401:db00:20:a01e:face:0:5:0]:10");
+ assertEquals(nid.getPort(), 10);
+ assertEquals(nid.getHost(), "[2401:db00:20:a01e:face:0:5:0]");
+
+ nid = ConverterUtils
+ .toNodeIdWithDefaultPort("[2401:db00:20:a01e:face:0:5:0]");
+ assertEquals(nid.getPort(), 0);
+ assertEquals(nid.getHost(), "[2401:db00:20:a01e:face:0:5:0]");
}
@Test(expected = IllegalArgumentException.class)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/lib/TestZKClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/lib/TestZKClient.java
index c2a86fc..fb7a481 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/lib/TestZKClient.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/lib/TestZKClient.java
@@ -27,6 +27,7 @@ import java.net.InetSocketAddress;
import java.net.Socket;
import org.apache.hadoop.net.ServerSocketUtil;
+import com.google.common.net.HostAndPort;
import org.junit.Assert;
import org.apache.hadoop.yarn.lib.ZKClient;
@@ -86,8 +87,9 @@ public class TestZKClient {
long start = System.currentTimeMillis();
while (true) {
try {
- String host = hp.split(":")[0];
- int port = Integer.parseInt(hp.split(":")[1]);
+ HostAndPort hap = HostAndPort.fromString(hp);
+ String host = hap.getHost();
+ int port = hap.getPort();
send4LetterWord(host, port, "stat");
} catch (IOException e) {
return true;
@@ -110,8 +112,9 @@ public class TestZKClient {
long start = System.currentTimeMillis();
while (true) {
try {
- String host = hp.split(":")[0];
- int port = Integer.parseInt(hp.split(":")[1]);
+ HostAndPort hap = HostAndPort.fromString(hp);
+ String host = hap.getHost();
+ int port = hap.getPort();
// if there are multiple hostports, just take the first one
String result = send4LetterWord(host, port, "stat");
if (result.startsWith("Zookeeper version:")) {
@@ -151,14 +154,15 @@ public class TestZKClient {
}
File dataDir = createTmpDir(BASETEST);
zks = new ZooKeeperServer(dataDir, dataDir, 3000);
- final int PORT = Integer.parseInt(hostPort.split(":")[1]);
+ HostAndPort hp = HostAndPort.fromString(hostPort);
+ final int port = hp.getPort();
if (factory == null) {
factory = new NIOServerCnxnFactory();
- factory.configure(new InetSocketAddress(PORT), maxCnxns);
+ factory.configure(new InetSocketAddress(port), maxCnxns);
}
factory.startup(zks);
Assert.assertTrue("waiting for server up",
- waitForServerUp("127.0.0.1:" + PORT,
+ waitForServerUp("127.0.0.1:" + port,
CONNECTION_TIMEOUT));
}
@@ -172,10 +176,11 @@ public class TestZKClient {
zkDb.close();
} catch (IOException ie) {
}
- final int PORT = Integer.parseInt(hostPort.split(":")[1]);
+ HostAndPort hp = HostAndPort.fromString(hostPort);
+ final int port = hp.getPort();
Assert.assertTrue("waiting for server down",
- waitForServerDown("127.0.0.1:" + PORT,
+ waitForServerDown("127.0.0.1:" + port,
CONNECTION_TIMEOUT));
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index ddfcdd8..5dfc91e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -185,6 +185,7 @@ import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import com.google.common.net.HostAndPort;
import static org.apache.hadoop.service.Service.STATE.STARTED;
@@ -645,7 +646,7 @@ public class ContainerManagerImpl extends CompositeService
implements
//hostname found when querying for our hostname with the specified
//address, combine the specified address with the actual port listened
//on by the server
- hostOverride = nmAddress.split(":")[0];
+ hostOverride = HostAndPort.fromString(nmAddress).getHost();
}
// setup node ID
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index a813a85..7ede033 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -137,6 +137,7 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
import org.eclipse.jetty.webapp.WebAppContext;
+import com.google.common.net.HostAndPort;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
@@ -1399,8 +1400,8 @@ public class ResourceManager extends CompositeService
builder.withAttribute(WebAppProxy.PROXY_CA,
rmContext.getProxyCAManager().getProxyCA());
builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher);
- String[] proxyParts = proxyHostAndPort.split(":");
- builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, proxyParts[0]);
+ builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE,
+ HostAndPort.fromString(proxyHostAndPort).getHost());
}
WebAppContext uiWebAppContext = null;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
index d433753..7f18c31 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
@@ -30,6 +30,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import com.google.common.net.HostAndPort;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
@@ -105,8 +106,10 @@ public class MockNM {
this.capability = capability;
this.resourceTracker = resourceTracker;
this.version = version;
- String[] splits = nodeIdStr.split(":");
- nodeId = BuilderUtils.newNodeId(splits[0], Integer.parseInt(splits[1]));
+ HostAndPort hostAndPort = HostAndPort.fromString(nodeIdStr);
+ String hostPortStr = hostAndPort.toString();
+ String host = hostPortStr.substring(0, hostPortStr.lastIndexOf(":"));
+ nodeId = BuilderUtils.newNodeId(host, hostAndPort.getPort());
}
public MockNM(String nodeIdStr, Resource capability,
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
index f373e94..eaf9360 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.webproxy;
import java.io.IOException;
import java.net.URI;
+import com.google.common.net.HostAndPort;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.authorize.AccessControlList;
@@ -57,7 +58,8 @@ public class WebAppProxy extends AbstractService {
@Override
protected void serviceInit(Configuration conf) throws Exception {
- String auth =
conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);
+ String auth = conf.get(
+ CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);
if (auth == null || "simple".equals(auth)) {
isSecurityEnabled = false;
} else if ("kerberos".equals(auth)) {
@@ -68,8 +70,7 @@ public class WebAppProxy extends AbstractService {
" of " + auth);
}
String proxy = WebAppUtils.getProxyHostAndPort(conf);
- String[] proxyParts = proxy.split(":");
- proxyHost = proxyParts[0];
+ proxyHost = HostAndPort.fromString(proxy).getHost();
fetcher = new AppReportFetcher(conf);
bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
index c023aae..1ac7cab 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.webproxy.amfilter;
import
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import com.google.common.net.HostAndPort;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.FilterContainer;
import org.apache.hadoop.http.FilterInitializer;
@@ -38,14 +39,15 @@ public class AmFilterInitializer extends FilterInitializer {
private static final String FILTER_NAME = "AM_PROXY_FILTER";
private static final String FILTER_CLASS =
AmIpFilter.class.getCanonicalName();
public static final String RM_HA_URLS = "RM_HA_URLS";
-
+
@Override
public void initFilter(FilterContainer container, Configuration conf) {
Map<String, String> params = new HashMap<>();
List<String> proxies = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf);
StringBuilder sb = new StringBuilder();
for (String proxy : proxies) {
- sb.append(proxy.split(":")[0]).append(AmIpFilter.PROXY_HOSTS_DELIMITER);
+ sb.append(HostAndPort.fromString(proxy).getHost())
+ .append(AmIpFilter.PROXY_HOSTS_DELIMITER);
}
sb.setLength(sb.length() - 1);
params.put(AmIpFilter.PROXY_HOSTS, sb.toString());
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
index 6c8993f..65121a6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
@@ -50,6 +50,7 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import com.google.common.net.HostAndPort;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.http.HttpServer2;
@@ -589,8 +590,7 @@ public class TestWebAppProxyServlet {
proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.TRUE);
String proxy = WebAppUtils.getProxyHostAndPort(conf);
- String[] proxyParts = proxy.split(":");
- String proxyHost = proxyParts[0];
+ String proxyHost = HostAndPort.fromString(proxy).getHost();
proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
proxyServer.start();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
