SLIDER-1121 fix slider AM port allocation race condition
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/55dd69dd Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/55dd69dd Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/55dd69dd Branch: refs/heads/feature/SLIDER-1107_AM_config_generation Commit: 55dd69dd36e743e645a67a53429c185dc4612307 Parents: 4b7d5c8 Author: Billie Rinaldi <billie.rina...@gmail.com> Authored: Tue May 17 06:57:29 2016 -0700 Committer: Billie Rinaldi <billie.rina...@gmail.com> Committed: Tue May 17 06:57:29 2016 -0700 ---------------------------------------------------------------------- .../org/apache/slider/common/SliderKeys.java | 5 -- .../apache/slider/common/tools/PortScanner.java | 16 +---- .../apache/slider/common/tools/SliderUtils.java | 16 +++++ .../server/appmaster/SliderAppMaster.java | 63 ++++++++------------ .../appmaster/web/rest/agent/AgentWebApp.java | 35 ++++++++++- 5 files changed, 79 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/55dd69dd/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 1d2d5f8..05c7048 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -266,11 +266,6 @@ public interface SliderKeys extends SliderXmlConfKeys { * {@value} */ String KEY_ALLOWED_PORT_RANGE = "site.global.slider.allowed.ports"; - - /** - * Allowed port range - */ - String KEY_AM_ALLOWED_PORT_RANGE = "slider.am.allowed.port.range"; /** * env var for custom JVM options. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/55dd69dd/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java b/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java index b5b21ce..5b80f9f 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java @@ -19,6 +19,7 @@ package org.apache.slider.common.tools; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.core.exceptions.SliderException; +import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -39,8 +40,6 @@ public class PortScanner { public PortScanner() { } - int nextPort = 1024; - public void setPortRange(String input) { // first split based on commas Set<Integer> inputPorts= new TreeSet<Integer>(); @@ -68,23 +67,14 @@ public class PortScanner { return remainingPortsToCheck; } - public int getAvailablePort() throws SliderException { + public int getAvailablePort() throws SliderException, IOException { if (remainingPortsToCheck != null) { return getAvailablePortViaPortArray(); } else { - return getAvailablePortViaCounter(); + return SliderUtils.getOpenPort(); } } - private int getAvailablePortViaCounter() throws SliderException { - int port; - do { - port = nextPort; - nextPort++; - } while (!SliderUtils.isPortAvailable(port)); - return port; - } - private int getAvailablePortViaPortArray() throws SliderException { boolean found = false; int availablePort = -1; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/55dd69dd/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index eae80f5..746e468 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -1114,6 +1114,22 @@ public final class SliderUtils { } /** + * Get a random open port + * @return true if the port was available for listening on + */ + public static int getOpenPort() throws IOException { + ServerSocket socket = null; + try { + socket = new ServerSocket(0); + return socket.getLocalPort(); + } finally { + if (socket != null) { + socket.close(); + } + } + } + + /** * See if a port is available for listening on by trying to listen * on it and seeing if that works or fails. * @param port port to listen to http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/55dd69dd/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 8d30da7..0776a6c 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -404,11 +404,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService private SecurityConfiguration securityConfiguration; /** - * The port for the web application - */ - private int webAppPort; - - /** * Is security enabled? * Set early on in the {@link #createAndRunCluster(String)} operation. */ @@ -776,11 +771,23 @@ public class SliderAppMaster extends AbstractSliderLaunchedService uploadServerCertForLocalization(clustername, fs); } - webAppPort = getPortToRequest(); - if (webAppPort == 0) { - // failure to find a port - throw new BadConfigException("Failed to fix a web application port"); - } + // Web service endpoints: initialize + WebAppApiImpl webAppApi = + new WebAppApiImpl( + stateForProviders, + providerService, + certificateManager, + registryOperations, + metricsAndMonitoring, + actionQueues, + this, + contentCache); + initAMFilterOptions(serviceConf); + + // start the agent web app + startAgentWebApp(appInformation, serviceConf, webAppApi); + int webAppPort = deployWebApplication(webAppApi); + String scheme = WebAppUtils.HTTP_PREFIX; appMasterTrackingUrl = scheme + appMasterHostname + ":" + webAppPort; @@ -926,7 +933,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService Path tmpDirPath = new Path(amTmpDir); Path launcherTmpDirPath = new Path(tmpDirPath, rolesTmpSubdir); fs.getFileSystem().mkdirs(launcherTmpDirPath); - + //launcher service launchService = new RoleLaunchService(actionQueues, providerService, @@ -972,25 +979,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService scheduleEscalation(instanceDefinition.getInternal()); try { - - // Web service endpoints: initialize - - WebAppApiImpl webAppApi = - new WebAppApiImpl( - stateForProviders, - providerService, - certificateManager, - registryOperations, - metricsAndMonitoring, - actionQueues, - this, - contentCache); - initAMFilterOptions(serviceConf); - - // start the agent web app - startAgentWebApp(appInformation, serviceConf, webAppApi); - deployWebApplication(webAppPort, webAppApi); - // schedule YARN Registry registration queue(new ActionRegisterServiceInstance(clustername, appid)); @@ -1051,7 +1039,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService } /** - * List the node reports: uses {@link #yarnClient} as the login user + * List the node reports: uses {@link SliderYarnClientImpl} as the login user * @param yarnClient client to the RM * @return the node reports * @throws IOException @@ -1083,17 +1071,18 @@ public class SliderAppMaster extends AbstractSliderLaunchedService * Creates and starts the web application, and adds a * <code>WebAppService</code> service under the AM, to ensure * a managed web application shutdown. - * @param port port to deploy the web application on * @param webAppApi web app API instance + * @return port the web application is deployed on * @throws IOException general problems starting the webapp (network, etc) * @throws WebAppException other issues */ - private void deployWebApplication(int port, WebAppApiImpl webAppApi) - throws IOException { + private int deployWebApplication(WebAppApiImpl webAppApi) + throws IOException, SliderException { try { webApp = new SliderAMWebApp(webAppApi); HttpConfig.Policy policy = HttpConfig.Policy.HTTP_ONLY; + int port = getPortToRequest(); log.info("Launching web application at port {} with policy {}", port, policy); WebApps.$for(SliderAMWebApp.BASE_PATH, @@ -1101,7 +1090,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService webAppApi, RestPaths.WS_CONTEXT) .withHttpPolicy(getConfig(), policy) - .at(port) + .at("0.0.0.0", port, true) .inDevMode() .start(webApp); @@ -1109,6 +1098,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService new WebAppService<>("slider", webApp); deployChildService(webAppService); + return webApp.port(); } catch (WebAppException e) { if (e.getCause() instanceof IOException) { throw (IOException)e.getCause(); @@ -1167,8 +1157,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService * @return the port to request. * @throws SliderException */ - private int getPortToRequest() - throws SliderException { + private int getPortToRequest() throws SliderException, IOException { return portScanner.getAvailablePort(); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/55dd69dd/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java index 200fbc2..3a3b0c0 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java @@ -25,7 +25,6 @@ import com.sun.jersey.spi.inject.SingletonTypeInjectableProvider; import org.apache.slider.core.conf.MapOperations; import org.apache.slider.providers.agent.AgentKeys; import org.apache.slider.server.appmaster.web.WebAppApi; -import org.apache.slider.server.appmaster.web.rest.RestPaths; import org.apache.slider.server.services.security.SecurityUtils; import org.mortbay.jetty.Connector; import org.mortbay.jetty.Server; @@ -40,6 +39,7 @@ import javax.ws.rs.ext.Provider; import java.io.Closeable; import java.io.File; import java.io.IOException; +import java.net.BindException; import java.util.Set; /** @@ -91,6 +91,7 @@ public class AgentWebApp implements Closeable { new QueuedThreadPool( configsMap.getOptionInt("agent.threadpool.size.max", 25))); agentServer.setStopAtShutdown(true); + agentServer.setGracefulShutdown(1000); SslSelectChannelConnector ssl1WayConnector = createSSLConnector(false, port); SslSelectChannelConnector ssl2WayConnector = @@ -115,6 +116,7 @@ public class AgentWebApp implements Closeable { agentRoot.addServlet(agent, "/*"); try { + openListeners(); agentServer.start(); } catch (IOException e) { LOG.error("Unable to start agent server", e); @@ -131,6 +133,37 @@ public class AgentWebApp implements Closeable { } + private void openListeners() throws Exception { + // from HttpServer2.openListeners() + for (Connector listener : agentServer.getConnectors()) { + if (listener.getLocalPort() != -1) { + // This listener is either started externally or has been bound + continue; + } + int port = listener.getPort(); + while (true) { + // jetty has a bug where you can't reopen a listener that previously + // failed to open w/o issuing a close first, even if the port is changed + try { + listener.close(); + listener.open(); + LOG.info("Jetty bound to port " + listener.getLocalPort()); + break; + } catch (BindException ex) { + if (port == 0) { + BindException be = new BindException("Port in use: " + + listener.getHost() + ":" + listener.getPort()); + be.initCause(ex); + throw be; + } + } + // try the next port number + listener.setPort(++port); + Thread.sleep(100); + } + } + } + private SslSelectChannelConnector createSSLConnector(boolean needClientAuth, int port) { SslSelectChannelConnector sslConnector = new SslSelectChannelConnector();
