SLIDER-947 build node map from yarn update reports; serve via REST/IPC âthis is done with YarnClient and building the initial map at launch time. Tests do not yet do this.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5887dde8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5887dde8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5887dde8 Branch: refs/heads/feature/SLIDER-82-pass-3.1 Commit: 5887dde8e68b5d650d692d6e1211a7382465c962 Parents: f1bad85 Author: Steve Loughran <[email protected]> Authored: Thu Nov 5 14:15:54 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Thu Nov 5 14:15:54 2015 +0000 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 21 +++++++++++--------- .../core/launch/JavaCommandLineBuilder.java | 17 ++++++++++++++-- .../server/appmaster/SliderAppMaster.java | 1 + .../slider/server/appmaster/state/AppState.java | 5 ++++- .../appmaster/state/AppStateBindingInfo.java | 1 + .../server/appmaster/state/RoleHistory.java | 7 ++++--- .../slider/agent/rest/TestStandaloneREST.groovy | 7 +++---- 7 files changed, 40 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5887dde8/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 07c915c..ea6810b 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -2103,9 +2103,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemBinding()); } - /** - * pass the registry binding - */ + // pass the registry binding commandLine.addConfOptionToCLI(config, RegistryConstants.KEY_REGISTRY_ZK_ROOT, RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT); commandLine.addMandatoryConfOption(config, RegistryConstants.KEY_REGISTRY_ZK_QUORUM); @@ -2115,6 +2113,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // the relevant security settings go over commandLine.addConfOption(config, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); } + + // copy over any/all YARN RM client values, in case the server-side XML conf file + // has the 0.0.0.0 address + commandLine.addConfOptions(config, + YarnConfiguration.RM_ADDRESS, + YarnConfiguration.RM_CLUSTER_ID, + YarnConfiguration.RM_HOSTNAME, + YarnConfiguration.RM_PRINCIPAL); + // write out the path output commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM); @@ -2129,12 +2136,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // Set the priority for the application master - - int amPriority = config.getInt(KEY_YARN_QUEUE_PRIORITY, - DEFAULT_YARN_QUEUE_PRIORITY); - - - amLauncher.setPriority(amPriority); + amLauncher.setPriority(config.getInt(KEY_YARN_QUEUE_PRIORITY, + DEFAULT_YARN_QUEUE_PRIORITY)); // Set the queue to which this application is to be submitted in the RM // Queue for App master http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5887dde8/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java index 9197e5d..ccb610a 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java @@ -25,6 +25,8 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.BadConfigException; +import java.util.Map; + /** * Command line builder purely for the Java CLI. * Some of the <code>define</code> methods are designed to work with Hadoop tool and @@ -86,8 +88,18 @@ public class JavaCommandLineBuilder extends CommandLineBuilder { } public boolean addConfOption(Configuration conf, String key) { - String val = conf.get(key); - return defineIfSet(key, val); + return defineIfSet(key, conf.get(key)); + } + + /** + * Add a varargs list of configuration parameters âif they are present + * @param conf configuration source + * @param keys keys + */ + public void addConfOptions(Configuration conf, String...keys) { + for (String key : keys) { + addConfOption(conf, key); + } } public String addConfOptionToCLI(Configuration conf, @@ -137,4 +149,5 @@ public class JavaCommandLineBuilder extends CommandLineBuilder { throw new BadConfigException("Missing configuration option: " + key); } } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5887dde8/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 1a127cf..d74688b 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -666,6 +666,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(serviceConf); if (!SliderUtils.isAddressDefined(clientRpcAddress)) { // client addr is being unset. We can lift it from the other RM APIs + log.warn("Yarn RM address was unbound; attempting to fix up"); serviceConf.set(YarnConfiguration.RM_ADDRESS, String.format("%s:%d", rmSchedulerAddress.getHostString(), clientRpcAddress.getPort() )); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5887dde8/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 1325148..9e29af2 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -541,7 +541,7 @@ public class AppState { //set the livespan MapOperations globalResOpts = instanceDefinition.getResourceOperations().getGlobalOptions(); - + startTimeThreshold = globalResOpts.getOptionInt( InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE, InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE); @@ -559,6 +559,9 @@ public class AppState { roleHistory = new RoleHistory(roleList); roleHistory.register(metricsAndMonitoring); roleHistory.onStart(binding.fs, binding.historyPath); + // trigger first node update + roleHistory.onNodesUpdated(binding.nodeReports); + //rebuild any live containers rebuildModelFromRestart(binding.liveContainers); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5887dde8/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java index a2a0b60..a4a9b7e 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java @@ -58,5 +58,6 @@ public class AppStateBindingInfo { Preconditions.checkArgument(roles != null, "null providerRoles"); Preconditions.checkArgument(fs != null, "null fs"); Preconditions.checkArgument(historyPath != null, "null historyDir"); + Preconditions.checkArgument(nodeReports != null, "null nodeReports"); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5887dde8/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index d9a6b34..53c2689 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -76,7 +76,8 @@ public class RoleHistory { /** Time when saved */ private final Timestamp saveTime = new Timestamp(0); - /** If the history was loaded, the time at which the history was saved */ + /** If the history was loaded, the time at which the history was saved. + * That is: the time the data was valid */ private final Timestamp thawedDataTime = new Timestamp(0); private NodeMap nodemap; @@ -817,12 +818,12 @@ public class RoleHistory { if (hostname == null || nodeState == null) { continue; } + log.debug("host {} is in state {}", hostname, nodeState); // update the node; this also creates an instance if needed boolean updated = nodemap.updateNode(hostname, updatedNode); if (updated) { - log.debug("Updated host {} to state {}", hostname, nodeState); if (nodeState.isUnusable()) { - log.info("Failed node {}", hostname); + log.info("Failed node {} state {}", hostname, nodeState); failedNodes.add(hostname); } else { failedNodes.remove(hostname); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5887dde8/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy index 29fa51a..7cb1837 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy @@ -71,7 +71,7 @@ class TestStandaloneREST extends AgentMiniClusterTestBase { execOperation(WEB_STARTUP_TIME) { GET(directAM) } - + execOperation(WEB_STARTUP_TIME) { def metrics = GET(directAM, SYSTEM_METRICS_JSON) log.info prettyPrintJson(metrics) @@ -84,14 +84,13 @@ class TestStandaloneREST extends AgentMiniClusterTestBase { log.info GET(proxyAM, SYSTEM_HEALTHCHECK) log.info GET(proxyAM, SYSTEM_METRICS_JSON) - // using the metrics, await the first node status update - /* SLIDER--82: disabled + // using the metrics, await the first node status update. + // this should be from AM launch itself awaitGaugeValue( appendToURL(proxyAM, SYSTEM_METRICS_JSON), "org.apache.slider.server.appmaster.state.RoleHistory.nodes-updated.flag", 1, WEB_STARTUP_TIME * 2, 500) - */ // Is the back door required? If so, don't test complex verbs via the proxy def proxyComplexVerbs = !SliderXmlConfKeys.X_DEV_INSECURE_REQUIRED
