Repository: hadoop
Updated Branches:
refs/heads/branch-2 378d62663 -> b7d59e790
HADOOP-12097. Allow port range to be specified while starting webapp.
Contributed by Varun Saxena.
(cherry picked from commit cce35c38159b23eb55204b3c9afcaa3215f4f4ef)
Conflicts:
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7d59e79
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7d59e79
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7d59e79
Branch: refs/heads/branch-2
Commit: b7d59e790b694219def055f1999c81e9cf9c1dc7
Parents: 378d626
Author: Junping Du <[email protected]>
Authored: Sun Feb 5 19:42:11 2017 -0800
Committer: Junping Du <[email protected]>
Committed: Sun Feb 5 20:13:49 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 12 ++
.../org/apache/hadoop/http/HttpServer2.java | 119 ++++++++++++++++---
.../org/apache/hadoop/http/TestHttpServer.java | 38 ++++++
.../org/apache/hadoop/yarn/webapp/WebApps.java | 38 ++++--
.../apache/hadoop/yarn/webapp/TestWebApp.java | 46 +++++++
5 files changed, 227 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7d59e79/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index a9675bf..005984d 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -1808,6 +1808,18 @@ public class Configuration implements
Iterable<Map.Entry<String,String>>,
return result.toString();
}
+ /**
+ * Get range start for the first integer range.
+ * @return range start.
+ */
+ public int getRangeStart() {
+ if (ranges == null || ranges.isEmpty()) {
+ return -1;
+ }
+ Range r = ranges.get(0);
+ return r.start;
+ }
+
@Override
public Iterator<Integer> iterator() {
return new RangeNumberIterator(ranges);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7d59e79/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
----------------------------------------------------------------------
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
index 570595be..5530208 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.ConfServlet;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.security.AuthenticationFilterInitializer;
import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
@@ -133,6 +134,7 @@ public final class HttpServer2 implements FilterContainer {
protected final WebAppContext webAppContext;
protected final boolean findPort;
+ protected final IntegerRanges portRanges;
protected final Map<Context, Boolean> defaultContexts =
new HashMap<>();
protected final List<String> filterNames = new ArrayList<>();
@@ -170,6 +172,7 @@ public final class HttpServer2 implements FilterContainer {
private String keyPassword;
private boolean findPort;
+ private IntegerRanges portRanges = null;
private String hostName;
private boolean disallowFallbackToRandomSignerSecretProvider;
@@ -242,6 +245,11 @@ public final class HttpServer2 implements FilterContainer {
return this;
}
+ public Builder setPortRanges(IntegerRanges ranges) {
+ this.portRanges = ranges;
+ return this;
+ }
+
public Builder setConf(Configuration conf) {
this.conf = conf;
return this;
@@ -397,6 +405,7 @@ public final class HttpServer2 implements FilterContainer {
}
this.findPort = b.findPort;
+ this.portRanges = b.portRanges;
initializeWebServer(b.name, b.hostName, b.conf, b.pathSpecs);
}
@@ -978,6 +987,93 @@ public final class HttpServer2 implements FilterContainer {
}
/**
+ * Bind listener by closing and opening the listener.
+ * @param listener
+ * @throws Exception
+ */
+ private static void bindListener(Connector listener) throws Exception {
+ // jetty has a bug where you can't reopen a listener that previously
+ // failed to open w/o issuing a close first, even if the port is changed
+ listener.close();
+ listener.open();
+ LOG.info("Jetty bound to port " + listener.getLocalPort());
+ }
+
+ /**
+ * Create bind exception by wrapping the bind exception thrown.
+ * @param listener
+ * @param ex
+ * @return
+ */
+ private static BindException constructBindException(Connector listener,
+ BindException ex) {
+ BindException be = new BindException("Port in use: "
+ + listener.getHost() + ":" + listener.getPort());
+ if (ex != null) {
+ be.initCause(ex);
+ }
+ return be;
+ }
+
+ /**
+ * Bind using single configured port. If findPort is true, we will try to
bind
+ * after incrementing port till a free port is found.
+ * @param listener jetty listener.
+ * @param port port which is set in the listener.
+ * @throws Exception
+ */
+ private void bindForSinglePort(Connector listener, int port)
+ throws Exception {
+ while (true) {
+ try {
+ bindListener(listener);
+ break;
+ } catch (BindException ex) {
+ if (port == 0 || !findPort) {
+ throw constructBindException(listener, ex);
+ }
+ }
+ // try the next port number
+ listener.setPort(++port);
+ Thread.sleep(100);
+ }
+ }
+
+ /**
+ * Bind using port ranges. Keep on looking for a free port in the port range
+ * and throw a bind exception if no port in the configured range binds.
+ * @param listener jetty listener.
+ * @param startPort initial port which is set in the listener.
+ * @throws Exception
+ */
+ private void bindForPortRange(Connector listener, int startPort)
+ throws Exception {
+ BindException bindException = null;
+ try {
+ bindListener(listener);
+ return;
+ } catch (BindException ex) {
+ // Ignore exception.
+ bindException = ex;
+ }
+ for(Integer port : portRanges) {
+ if (port == startPort) {
+ continue;
+ }
+ Thread.sleep(100);
+ listener.setPort(port);
+ try {
+ bindListener(listener);
+ return;
+ } catch (BindException ex) {
+ // Ignore exception. Move to next port.
+ bindException = ex;
+ }
+ }
+ throw constructBindException(listener, bindException);
+ }
+
+ /**
* Open the main listener for the server
* @throws Exception
*/
@@ -988,25 +1084,10 @@ public final class HttpServer2 implements
FilterContainer {
continue;
}
int port = listener.getPort();
- while (true) {
- // jetty has a bug where you can't reopen a listener that previously
- // failed to open w/o issuing a close first, even if the port is
changed
- try {
- listener.close();
- listener.open();
- LOG.info("Jetty bound to port " + listener.getLocalPort());
- break;
- } catch (BindException ex) {
- if (port == 0 || !findPort) {
- BindException be = new BindException("Port in use: "
- + listener.getHost() + ":" + listener.getPort());
- be.initCause(ex);
- throw be;
- }
- }
- // try the next port number
- listener.setPort(++port);
- Thread.sleep(100);
+ if (portRanges != null && port != 0) {
+ bindForPortRange(listener, port);
+ } else {
+ bindForSinglePort(listener, port);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7d59e79/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
----------------------------------------------------------------------
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
index f60ad68..b8c5e11 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
@@ -20,10 +20,12 @@ package org.apache.hadoop.http;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.http.HttpServer2.QuotingInputFilter.RequestQuoter;
import org.apache.hadoop.http.resource.JerseyResource;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
import org.apache.hadoop.security.UserGroupInformation;
@@ -638,4 +640,40 @@ public class TestHttpServer extends
HttpServerFunctionalTest {
assertNotNull(conn.getHeaderField("Date"));
assertEquals(conn.getHeaderField("Expires"), conn.getHeaderField("Date"));
}
+
+ private static void stopHttpServer(HttpServer2 server) throws Exception {
+ if (server != null) {
+ server.stop();
+ }
+ }
+
+ @Test
+ public void testPortRanges() throws Exception {
+ Configuration conf = new Configuration();
+ int port = ServerSocketUtil.waitForPort(49000, 60);
+ int endPort = 49500;
+ conf.set("abc", "49000-49500");
+ HttpServer2.Builder builder = new HttpServer2.Builder()
+ .setName("test").setConf(new Configuration()).setFindPort(false);
+ IntegerRanges ranges = conf.getRange("abc", "");
+ int startPort = 0;
+ if (ranges != null && !ranges.isEmpty()) {
+ startPort = ranges.getRangeStart();
+ builder.setPortRanges(ranges);
+ }
+ builder.addEndpoint(URI.create("http://localhost:"; + startPort));
+ HttpServer2 myServer = builder.build();
+ HttpServer2 myServer2 = null;
+ try {
+ myServer.start();
+ assertEquals(port, myServer.getConnectorAddress(0).getPort());
+ myServer2 = builder.build();
+ myServer2.start();
+ assertTrue(myServer2.getConnectorAddress(0).getPort() > port &&
+ myServer2.getConnectorAddress(0).getPort() <= endPort);
+ } finally {
+ stopHttpServer(myServer);
+ stopHttpServer(myServer2);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7d59e79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
index 53cb3ee..9c96339 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
@@ -35,6 +35,7 @@ import javax.servlet.http.HttpServlet;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.http.HttpConfig.Policy;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.UserGroupInformation;
@@ -91,6 +92,7 @@ public class WebApps {
boolean findPort = false;
Configuration conf;
Policy httpPolicy = null;
+ String portRangeConfigKey = null;
boolean devMode = false;
private String spnegoPrincipalKey;
private String spnegoKeytabKey;
@@ -156,6 +158,19 @@ public class WebApps {
return this;
}
+ /**
+ * Set port range config key and associated configuration object.
+ * @param config configuration.
+ * @param portRangeConfKey port range config key.
+ * @return builder object.
+ */
+ public Builder<T> withPortRange(Configuration config,
+ String portRangeConfKey) {
+ this.conf = config;
+ this.portRangeConfigKey = portRangeConfKey;
+ return this;
+ }
+
public Builder<T> withHttpSpnegoPrincipalKey(String spnegoPrincipalKey) {
this.spnegoPrincipalKey = spnegoPrincipalKey;
return this;
@@ -264,15 +279,24 @@ public class WebApps {
: WebAppUtils.HTTP_PREFIX;
}
HttpServer2.Builder builder = new HttpServer2.Builder()
- .setName(name)
- .addEndpoint(
- URI.create(httpScheme + bindAddress
- + ":" + port)).setConf(conf).setFindPort(findPort)
+ .setName(name).setConf(conf).setFindPort(findPort)
.setACL(new AccessControlList(conf.get(
- YarnConfiguration.YARN_ADMIN_ACL,
- YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)))
+ YarnConfiguration.YARN_ADMIN_ACL,
+ YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)))
.setPathSpec(pathList.toArray(new String[0]));
-
+ // Get port ranges from config.
+ IntegerRanges ranges = null;
+ if (portRangeConfigKey != null) {
+ ranges = conf.getRange(portRangeConfigKey, "");
+ }
+ int startPort = port;
+ if (ranges != null && !ranges.isEmpty()) {
+ // Set port ranges if its configured.
+ startPort = ranges.getRangeStart();
+ builder.setPortRanges(ranges);
+ }
+ builder.addEndpoint(URI.create(httpScheme + bindAddress +
+ ":" + startPort));
boolean hasSpnegoConf = spnegoPrincipalKey != null
&& conf.get(spnegoPrincipalKey) != null && spnegoKeytabKey != null
&& conf.get(spnegoKeytabKey) != null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7d59e79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
index deef855..9454002 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
@@ -35,6 +35,8 @@ import java.net.HttpURLConnection;
import java.net.URL;
import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.webapp.view.HtmlPage;
import org.apache.hadoop.yarn.webapp.view.JQueryUI;
@@ -307,6 +309,50 @@ public class TestWebApp {
}
}
+ private static void stopWebApp(WebApp app) {
+ if (app != null) {
+ app.stop();
+ }
+ }
+
+ @Test
+ public void testPortRanges() throws Exception {
+ WebApp app = WebApps.$for("test", this).start();
+ String baseUrl = baseUrl(app);
+ WebApp app1 = null;
+ WebApp app2 = null;
+ WebApp app3 = null;
+ WebApp app4 = null;
+ WebApp app5 = null;
+ try {
+ int port = ServerSocketUtil.waitForPort(48000, 60);
+ assertEquals("foo", getContent(baseUrl +"test/foo").trim());
+ app1 = WebApps.$for("test", this).at(port).start();
+ assertEquals(port, app1.getListenerAddress().getPort());
+ app2 = WebApps.$for("test", this).at("0.0.0.0",port, true).start();
+ assertTrue(app2.getListenerAddress().getPort() > port);
+ Configuration conf = new Configuration();
+ port = ServerSocketUtil.waitForPort(47000, 60);
+ app3 = WebApps.$for("test", this).at(port).withPortRange(conf, "abc").
+ start();
+ assertEquals(port, app3.getListenerAddress().getPort());
+ ServerSocketUtil.waitForPort(46000, 60);
+ conf.set("abc", "46000-46500");
+ app4 = WebApps.$for("test", this).at(port).withPortRange(conf, "abc").
+ start();
+ assertEquals(46000, app4.getListenerAddress().getPort());
+ app5 = WebApps.$for("test", this).withPortRange(conf, "abc").start();
+ assertTrue(app5.getListenerAddress().getPort() > 46000);
+ } finally {
+ stopWebApp(app);
+ stopWebApp(app1);
+ stopWebApp(app2);
+ stopWebApp(app3);
+ stopWebApp(app4);
+ stopWebApp(app5);
+ }
+ }
+
static String baseUrl(WebApp app) {
return "http://localhost:"+ app.port() +"/";
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
