YARN-9130. Add Bind_HOST configuration for Yarn Web Proxy. Contributed by Rong Tang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5df9fb16 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5df9fb16 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5df9fb16 Branch: refs/heads/HDFS-12943 Commit: 5df9fb16b9871618f19f3081b7006d3e77106737 Parents: 499c70e Author: Inigo Goiri <[email protected]> Authored: Wed Dec 19 18:58:00 2018 -0800 Committer: Inigo Goiri <[email protected]> Committed: Wed Dec 19 18:58:00 2018 -0800 ---------------------------------------------------------------------- .../hadoop/yarn/conf/YarnConfiguration.java | 6 +++- .../src/main/resources/yarn-default.xml | 9 ++++++ .../yarn/server/webproxy/WebAppProxy.java | 14 +++++++-- .../yarn/server/webproxy/WebAppProxyServer.java | 10 ++++--- .../server/webproxy/TestWebAppProxyServer.java | 31 ++++++++++++++++---- 5 files changed, 57 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df9fb16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 72cc0d6..d84d11a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2302,13 +2302,17 @@ public class YarnConfiguration extends Configuration { /** Keytab for Proxy.*/ public static final String PROXY_KEYTAB = PROXY_PREFIX + "keytab"; - + /** The address for the web proxy.*/ public static final String PROXY_ADDRESS = PROXY_PREFIX + "address"; public static final int DEFAULT_PROXY_PORT = 9099; public static final String DEFAULT_PROXY_ADDRESS = "0.0.0.0:" + DEFAULT_PROXY_PORT; + + /** Binding address for the web proxy. */ + public static final String PROXY_BIND_HOST = + PROXY_PREFIX + "bind-host"; /** * YARN Service Level Authorization http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df9fb16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index ccceaab..12c5ff6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2157,6 +2157,15 @@ <value/> </property> + <property> + <description>The actual address the web proxy will bind to. If this optional + address is set, it overrides only the hostname portion of yarn.web-proxy.address. + This is useful for making the web proxy server listen on all interfaces by setting + it to 0.0.0.0 </description> + <name>yarn.web-proxy.bind-host</name> + <value/> + </property> + <!-- Applications' Configuration --> <property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df9fb16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java index f205cf7..1382fc6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java @@ -74,16 +74,26 @@ public class WebAppProxy extends AbstractService { fetcher = new AppReportFetcher(conf); bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS); if(bindAddress == null || bindAddress.isEmpty()) { - throw new YarnRuntimeException(YarnConfiguration.PROXY_ADDRESS + + throw new YarnRuntimeException(YarnConfiguration.PROXY_ADDRESS + " is not set so the proxy will not run."); } - LOG.info("Instantiating Proxy at " + bindAddress); + String[] parts = StringUtils.split(bindAddress, ':'); port = 0; if (parts.length == 2) { bindAddress = parts[0]; port = Integer.parseInt(parts[1]); } + + String bindHost = conf.getTrimmed(YarnConfiguration.PROXY_BIND_HOST, null); + if (bindHost != null) { + LOG.debug("{} is set, will be used to run proxy.", + YarnConfiguration.PROXY_BIND_HOST); + bindAddress = bindHost; + } + + LOG.info("Instantiating Proxy at {}:{}", bindAddress, port); + acl = new AccessControlList(conf.get(YarnConfiguration.YARN_ADMIN_ACL, YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); super.serviceInit(conf); http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df9fb16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java index a5dd8f6..6be6039 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServer.java @@ -38,7 +38,7 @@ import org.slf4j.LoggerFactory; /** * ProxyServer will sit in between the end user and AppMaster - * web interfaces. + * web interfaces. */ public class WebAppProxyServer extends CompositeService { @@ -103,9 +103,11 @@ public class WebAppProxyServer extends CompositeService { * @return InetSocketAddress */ public static InetSocketAddress getBindAddress(Configuration conf) { - return conf.getSocketAddr(YarnConfiguration.PROXY_ADDRESS, - YarnConfiguration.DEFAULT_PROXY_ADDRESS, - YarnConfiguration.DEFAULT_PROXY_PORT); + return conf.getSocketAddr( + YarnConfiguration.PROXY_BIND_HOST, + YarnConfiguration.PROXY_ADDRESS, + YarnConfiguration.DEFAULT_PROXY_ADDRESS, + YarnConfiguration.DEFAULT_PROXY_PORT); } public static void main(String[] args) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/5df9fb16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java index 93479a9..87907a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java @@ -23,7 +23,6 @@ import static org.junit.Assert.assertEquals; import org.apache.hadoop.service.Service; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -33,14 +32,15 @@ import java.net.InetSocketAddress; public class TestWebAppProxyServer { private WebAppProxyServer webAppProxy = null; - private final String proxyAddress = "0.0.0.0:8888"; + private final String port = "8888"; + private final String proxyAddress = "localhost:" + port; + private YarnConfiguration conf = null; @Before public void setUp() throws Exception { - YarnConfiguration conf = new YarnConfiguration(); + conf = new YarnConfiguration(); conf.set(YarnConfiguration.PROXY_ADDRESS, proxyAddress); webAppProxy = new WebAppProxyServer(); - webAppProxy.init(conf); } @After @@ -50,19 +50,38 @@ public class TestWebAppProxyServer { @Test public void testStart() { + webAppProxy.init(conf); assertEquals(STATE.INITED, webAppProxy.getServiceState()); webAppProxy.start(); for (Service service : webAppProxy.getServices()) { if (service instanceof WebAppProxy) { - assertEquals(((WebAppProxy) service).getBindAddress(), proxyAddress); + assertEquals(proxyAddress, ((WebAppProxy) service).getBindAddress()); } } assertEquals(STATE.STARTED, webAppProxy.getServiceState()); } @Test + public void testStartWithBindHost() { + String bindHost = "0.0.0.0"; + conf.set(YarnConfiguration.PROXY_BIND_HOST, bindHost); + webAppProxy.init(conf); + + assertEquals(STATE.INITED, webAppProxy.getServiceState()); + webAppProxy.start(); + for (Service service : webAppProxy.getServices()) { + if (service instanceof WebAppProxy) { + assertEquals(bindHost + ":" + port, + ((WebAppProxy) service).getBindAddress()); + } + } + assertEquals(STATE.STARTED, webAppProxy.getServiceState()); + } + + + @Test public void testBindAddress() { - YarnConfiguration conf = new YarnConfiguration(); + conf = new YarnConfiguration(); InetSocketAddress defaultBindAddress = WebAppProxyServer.getBindAddress(conf); Assert.assertEquals("Web Proxy default bind address port is incorrect", --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
