SLIDER-82 trying to set up YarnClient, having config propagation issues in minicluster (RM client addr is 0.0.0.0)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/dd607335 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/dd607335 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/dd607335 Branch: refs/heads/feature/SLIDER-82-pass-3 Commit: dd6073352d7c7b8690098fee19ecef5b9d4b321f Parents: c6231fe Author: Steve Loughran <[email protected]> Authored: Wed Nov 4 21:12:53 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Wed Nov 4 21:12:53 2015 +0000 ---------------------------------------------------------------------- .../slider/client/SliderYarnClientImpl.java | 8 +++--- .../slider/core/launch/ContainerLauncher.java | 12 +++------ .../server/appmaster/SliderAppMaster.java | 27 ++++++++++---------- 3 files changed, 23 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dd607335/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java index 803ccd6..867603b 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java @@ -45,6 +45,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.BindException; +import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; @@ -69,10 +70,11 @@ public class SliderYarnClientImpl extends YarnClientImpl { @Override protected void serviceInit(Configuration conf) throws Exception { - String addr = conf.get(YarnConfiguration.RM_ADDRESS); - if (addr.startsWith("0.0.0.0")) { + InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(conf); + if (!SliderUtils.isAddressDefined(clientRpcAddress)) { // address isn't known; fail fast - throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS + " value:" + addr + throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS + + " value:" + conf.get(YarnConfiguration.RM_ADDRESS) + " - see https://wiki.apache.org/hadoop/UnsetHostnameOrPort";); } super.serviceInit(conf); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dd607335/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java index f8ea4ee..69b937d 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java @@ -55,17 +55,13 @@ public class ContainerLauncher extends AbstractLauncher { public UserGroupInformation setupUGI() { UserGroupInformation user = UserGroupInformation.createRemoteUser(container.getId().toString()); - String cmIpPortStr = - container.getNodeId().getHost() + ":" + container.getNodeId().getPort(); - final InetSocketAddress cmAddress = - NetUtils.createSocketAddr(cmIpPortStr); + String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort(); + final InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr); - org.apache.hadoop.yarn.api.records.Token containerToken = - container.getContainerToken(); + org.apache.hadoop.yarn.api.records.Token containerToken = container.getContainerToken(); if (containerToken != null) { Token<ContainerTokenIdentifier> token = - ConverterUtils.convertFromYarn(containerToken, - cmAddress); + ConverterUtils.convertFromYarn(containerToken, cmAddress); user.addToken(token); } return user; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dd607335/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index e6a5bd5..1a127cf 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -502,15 +502,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService addService(executorService); addService(actionQueues); - // set up the YARN client. This may require patching in the RM client-API address if it - // is (somehow) unset server-side. - String clientRMaddr = conf.get(YarnConfiguration.RM_ADDRESS); - if (clientRMaddr.startsWith("0.0.0.0")) { - // address isn't known; fail fast - throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS + " value:" + addr - + " - see https://wiki.apache.org/hadoop/UnsetHostnameOrPort";); - } - addService(yarnClient = new SliderYarnClientImpl()); //init all child services super.serviceInit(conf); @@ -542,8 +533,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService * @param args argument list */ @Override // RunService - public Configuration bindArgs(Configuration config, String... args) throws - Exception { + public Configuration bindArgs(Configuration config, String... args) throws Exception { // let the superclass process it Configuration superConf = super.bindArgs(config, args); // add the slider XML config @@ -667,10 +657,21 @@ public class SliderAppMaster extends AbstractSliderLaunchedService sliderAMProvider = new SliderAMProviderService(); initAndAddService(sliderAMProvider); - InetSocketAddress address = SliderUtils.getRmSchedulerAddress(serviceConf); - log.info("RM is at {}", address); + InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf); + log.info("RM is at {}", rmSchedulerAddress); yarnRPC = YarnRPC.create(serviceConf); + // set up the YARN client. This may require patching in the RM client-API address if it + // is (somehow) unset server-side. String clientRMaddr = serviceConf.get(YarnConfiguration.RM_ADDRESS); + InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(serviceConf); + if (!SliderUtils.isAddressDefined(clientRpcAddress)) { + // client addr is being unset. We can lift it from the other RM APIs + serviceConf.set(YarnConfiguration.RM_ADDRESS, + String.format("%s:%d", rmSchedulerAddress.getHostString(), clientRpcAddress.getPort() )); + } + initAndAddService(yarnClient = new SliderYarnClientImpl()); + yarnClient.start(); + /* * Extract the container ID. This is then * turned into an (incompete) container
