YARN-6405. Improve configuring services through REST API. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d23a97d4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d23a97d4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d23a97d4 Branch: refs/heads/yarn-native-services Commit: d23a97d4efc8f2d2e45920cdb2c3ec68733078b8 Parents: 633eb66 Author: Billie Rinaldi <bil...@apache.org> Authored: Wed Apr 26 08:44:38 2017 -0700 Committer: Billie Rinaldi <bil...@apache.org> Committed: Wed Apr 26 08:44:38 2017 -0700 ---------------------------------------------------------------------- ...RN-Simplified-V1-API-Layer-For-Services.yaml | 4 +- .../api/impl/TestApplicationApiService.java | 26 +- .../apache/slider/api/ServiceApiConstants.java | 50 +++ .../apache/slider/api/resource/ConfigFile.java | 58 +-- .../slider/api/resource/Configuration.java | 39 +- .../org/apache/slider/client/SliderClient.java | 16 +- .../org/apache/slider/common/SliderKeys.java | 4 +- .../slider/core/launch/CommandLineBuilder.java | 15 - .../docstore/ConfigurationResolver.java | 24 -- .../apache/slider/providers/ProviderRole.java | 32 +- .../slider/providers/ProviderService.java | 4 +- .../apache/slider/providers/ProviderUtils.java | 279 +++++++++----- .../providers/docker/DockerProviderService.java | 50 +-- .../server/appmaster/RoleLaunchService.java | 24 +- .../server/appmaster/SliderAppMaster.java | 77 ++-- .../actions/RegisterComponentInstance.java | 12 +- .../actions/UnregisterComponentInstance.java | 16 +- .../server/appmaster/metrics/SliderMetrics.java | 23 ++ .../appmaster/monkey/ChaosKillContainer.java | 2 +- .../server/appmaster/rpc/SliderIPCService.java | 2 +- .../slider/server/appmaster/state/AppState.java | 384 ++++++++++--------- .../server/appmaster/state/RoleInstance.java | 38 +- .../server/appmaster/state/RoleStatus.java | 1 + .../state/StateAccessForProviders.java | 1 + .../slider/util/RestApiErrorMessages.java | 2 +- .../org/apache/slider/util/ServiceApiUtil.java | 77 +++- .../TestMockAppStateDynamicHistory.java | 8 +- .../TestMockAppStateFlexDynamicRoles.java | 6 +- .../appstate/TestMockAppStateUniqueNames.java | 77 +++- .../TestMockContainerResourceAllocations.java | 11 + .../model/mock/BaseMockAppStateTest.java | 9 +- .../model/mock/MockProviderService.java | 4 +- 32 files changed, 850 insertions(+), 525 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml index e9239e4..82cc30f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml @@ -347,10 +347,10 @@ definitions: description: The absolute path that this configuration file should be mounted as, in the application container. src_file: type: string - description: Required for type template. This provides the source location of the template which needs to be mounted as dest_file post property substitutions. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, etc. + description: This provides the source location of the configuration file, the content of which is dumped to dest_file post property substitutions, in the format as specified in type. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported. props: type: object - description: A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If the type is template then the attribute src_file is mandatory and the src_file content is dumped to dest_file post property substitutions. + description: A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any, existing properties in src_file or be added as new properties in src_file. Container: description: An instance of a running application container. properties: http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java index abad34e..6e077d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java @@ -63,7 +63,7 @@ public class TestApplicationApiService { // no name try { - ServiceApiUtil.validateApplicationPostPayload(app); + ServiceApiUtil.validateApplicationPayload(app, null); Assert.fail(EXCEPTION_PREFIX + "application with no name"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage()); @@ -74,7 +74,7 @@ public class TestApplicationApiService { for (String badName : badNames) { app.setName(badName); try { - ServiceApiUtil.validateApplicationPostPayload(app); + ServiceApiUtil.validateApplicationPayload(app, null); Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID_FORMAT, @@ -85,7 +85,7 @@ public class TestApplicationApiService { // no artifact app.setName("finance_home"); try { - ServiceApiUtil.validateApplicationPostPayload(app); + ServiceApiUtil.validateApplicationPayload(app, null); Assert.fail(EXCEPTION_PREFIX + "application with no artifact"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_ARTIFACT_INVALID, e.getMessage()); @@ -95,7 +95,7 @@ public class TestApplicationApiService { Artifact artifact = new Artifact(); app.setArtifact(artifact); try { - ServiceApiUtil.validateApplicationPostPayload(app); + ServiceApiUtil.validateApplicationPayload(app, null); Assert.fail(EXCEPTION_PREFIX + "application with no artifact id"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage()); @@ -106,7 +106,7 @@ public class TestApplicationApiService { artifact.setId("app.io/hbase:facebook_0.2"); app.setNumberOfContainers(5l); try { - ServiceApiUtil.validateApplicationPostPayload(app); + ServiceApiUtil.validateApplicationPayload(app, null); } catch (IllegalArgumentException e) { logger.error("application attributes specified should be valid here", e); Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); @@ -128,7 +128,7 @@ public class TestApplicationApiService { // resource not specified artifact.setId("docker.io/centos:centos7"); try { - ServiceApiUtil.validateApplicationPostPayload(app); + ServiceApiUtil.validateApplicationPayload(app, null); Assert.fail(EXCEPTION_PREFIX + "application with no resource"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_RESOURCE_INVALID, e.getMessage()); @@ -138,7 +138,7 @@ public class TestApplicationApiService { Resource res = new Resource(); app.setResource(res); try { - ServiceApiUtil.validateApplicationPostPayload(app); + ServiceApiUtil.validateApplicationPayload(app, null); Assert.fail(EXCEPTION_PREFIX + "application with no memory"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_RESOURCE_MEMORY_INVALID, e.getMessage()); @@ -149,7 +149,7 @@ public class TestApplicationApiService { res.setMemory("100mb"); res.setCpus(-2); try { - ServiceApiUtil.validateApplicationPostPayload(app); + ServiceApiUtil.validateApplicationPayload(app, null); Assert.fail( EXCEPTION_PREFIX + "application with invalid no of cpups"); } catch (IllegalArgumentException e) { @@ -159,17 +159,17 @@ public class TestApplicationApiService { // number of containers not specified res.setCpus(2); try { - ServiceApiUtil.validateApplicationPostPayload(app); + ServiceApiUtil.validateApplicationPayload(app, null); Assert.fail( EXCEPTION_PREFIX + "application with no container count"); } catch (IllegalArgumentException e) { - Assert.assertEquals(ERROR_CONTAINERS_COUNT_INVALID, e.getMessage()); + Assert.assertTrue(e.getMessage().contains(ERROR_CONTAINERS_COUNT_INVALID)); } // specifying profile along with cpus/memory raises exception res.setProfile("hbase_finance_large"); try { - ServiceApiUtil.validateApplicationPostPayload(app); + ServiceApiUtil.validateApplicationPayload(app, null); Assert.fail(EXCEPTION_PREFIX + "application with resource profile along with cpus/memory"); } catch (IllegalArgumentException e) { @@ -182,7 +182,7 @@ public class TestApplicationApiService { res.setCpus(null); res.setMemory(null); try { - ServiceApiUtil.validateApplicationPostPayload(app); + ServiceApiUtil.validateApplicationPayload(app, null); Assert.fail(EXCEPTION_PREFIX + "application with resource profile only - NOT SUPPORTED"); } catch (IllegalArgumentException e) { @@ -198,7 +198,7 @@ public class TestApplicationApiService { // everything valid here app.setNumberOfContainers(5l); try { - ServiceApiUtil.validateApplicationPostPayload(app); + ServiceApiUtil.validateApplicationPayload(app, null); } catch (IllegalArgumentException e) { logger.error("application attributes specified should be valid here", e); Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java new file mode 100644 index 0000000..5f76f19 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import static org.apache.slider.util.ServiceApiUtil.$; + +/** + * This class defines constants that can be used in input spec for + * variable substitutions + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public interface ServiceApiConstants { + + // Constants for service + String SERVICE_NAME = $("SERVICE_NAME"); + + String SERVICE_NAME_LC = $("SERVICE_NAME.lc"); + + // Constants for component + String COMPONENT_NAME = $("COMPONENT_NAME"); + + String COMPONENT_NAME_LC = $("COMPONENT_NAME.lc"); + + String COMPONENT_INSTANCE_NAME = $("COMPONENT_INSTANCE_NAME"); + + // Constants for component instance + String COMPONENT_ID = $("COMPONENT_ID"); + + String CONTAINER_ID = $("CONTAINER_ID"); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java index cdc96b8..b4040b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java @@ -17,20 +17,19 @@ package org.apache.slider.api.resource; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonValue; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; import java.io.Serializable; +import java.util.HashMap; import java.util.Map; import java.util.Objects; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlRootElement; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonValue; - /** * A config file that needs to be created and made available as a volume in an * application component container. @@ -45,7 +44,7 @@ public class ConfigFile implements Serializable { public enum TypeEnum { XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE( - "TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"); + "TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),; private String value; @@ -63,7 +62,18 @@ public class ConfigFile implements Serializable { private TypeEnum type = null; private String destFile = null; private String srcFile = null; - private Map<String, String> props = null; + private Map<String, String> props = new HashMap<>(); + + public ConfigFile copy() { + ConfigFile copy = new ConfigFile(); + copy.setType(this.getType()); + copy.setSrcFile(this.getSrcFile()); + copy.setDestFile(this.getDestFile()); + if (this.getProps() != null && !this.getProps().isEmpty()) { + copy.getProps().putAll(this.getProps()); + } + return copy; + } /** * Config file in the standard format like xml, properties, json, yaml, @@ -105,19 +115,20 @@ public class ConfigFile implements Serializable { } /** - * TODO this probably is not required for non-template configs. It is now used as symlink for localization for non-template configs - we could infer the name from destFile instead - * - * Required for type template. This provides the source location of the - * template which needs to be mounted as dest_file post property - * substitutions. Typically the src_file would point to a source controlled - * network accessible file maintained by tools like puppet, chef, etc. + * This provides the source location of the configuration file, the content + * of which is dumped to dest_file post property substitutions, in the format + * as specified in type. Typically the src_file would point to a source + * controlled network accessible file maintained by tools like puppet, chef, + * or hdfs etc. Currently, only hdfs is supported. **/ public ConfigFile srcFile(String srcFile) { this.srcFile = srcFile; return this; } - @ApiModelProperty(example = "null", value = "Required for type template. This provides the source location of the template which needs to be mounted as dest_file post property substitutions. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, etc.") + @ApiModelProperty(example = "null", value = "This provides the source location of the configuration file, " + + "the content of which is dumped to dest_file post property substitutions, in the format as specified in type. " + + "Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, or hdfs etc. Currently, only hdfs is supported.") @JsonProperty("src_file") public String getSrcFile() { return srcFile; @@ -129,17 +140,19 @@ public class ConfigFile implements Serializable { } /** - * A blob of key value pairs that will be dumped in the dest_file in the - * format as specified in type. If the type is template then the attribute - * src_file is mandatory and the src_file content is dumped to dest_file post - * property substitutions. + A blob of key value pairs that will be dumped in the dest_file in the format + as specified in type. If src_file is specified, src_file content are dumped + in the dest_file and these properties will overwrite, if any, existing + properties in src_file or be added as new properties in src_file. **/ public ConfigFile props(Map<String, String> props) { this.props = props; return this; } - @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If the type is template then the attribute src_file is mandatory and the src_file content is dumped to dest_file post property substitutions.") + @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be dumped in the dest_file in the format as specified in type." + + " If src_file is specified, src_file content are dumped in the dest_file and these properties will overwrite, if any," + + " existing properties in src_file or be added as new properties in src_file.") @JsonProperty("props") public Map<String, String> getProps() { return props; @@ -175,8 +188,7 @@ public class ConfigFile implements Serializable { ConfigFile configFile = (ConfigFile) o; return Objects.equals(this.type, configFile.type) && Objects.equals(this.destFile, configFile.destFile) - && Objects.equals(this.srcFile, configFile.srcFile) - && Objects.equals(this.props, configFile.props); + && Objects.equals(this.srcFile, configFile.srcFile); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java index 7b3b93e..0df586c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java @@ -17,8 +17,11 @@ package org.apache.slider.api.resource; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import org.apache.commons.lang.StringUtils; import java.io.Serializable; import java.util.ArrayList; @@ -27,10 +30,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.commons.lang.StringUtils; - /** * Set of configuration properties that can be injected into the application * components via envs, files and custom pluggable helper docker containers. @@ -156,6 +155,13 @@ public class Configuration implements Serializable { return properties.get(name.trim()); } + public String getEnv(String name) { + if (name == null) { + return null; + } + return env.get(name.trim()); + } + @Override public boolean equals(java.lang.Object o) { if (this == o) { @@ -198,4 +204,29 @@ public class Configuration implements Serializable { } return o.toString().replace("\n", "\n "); } + + /** + * Merge all properties and envs from that configuration to this configration. + * For ConfigFiles, all properties and envs of that ConfigFile are merged into + * this ConfigFile. + */ + public synchronized void mergeFrom(Configuration that) { + this.properties.putAll(that.getProperties()); + this.env.putAll(that.getEnv()); + Map<String, ConfigFile> thatMap = new HashMap<>(); + for (ConfigFile file : that.getFiles()) { + thatMap.put(file.getDestFile(), file.copy()); + } + for (ConfigFile thisFile : files) { + if(thatMap.containsKey(thisFile.getDestFile())) { + ConfigFile thatFile = thatMap.get(thisFile.getDestFile()); + thisFile.getProps().putAll(thatFile.getProps()); + thatMap.remove(thisFile.getDestFile()); + } + } + // add remaining new files from that Configration + for (ConfigFile thatFile : thatMap.values()) { + files.add(thatFile.copy()); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 2b0982f..7241374 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -653,7 +653,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe public ApplicationId actionCreate(Application application) throws IOException, YarnException { - ServiceApiUtil.validateApplicationPostPayload(application); + ServiceApiUtil.validateApplicationPayload(application, + sliderFileSystem.getFileSystem()); String appName = application.getName(); validateClusterName(appName); verifyNoLiveApp(appName, "Create"); @@ -692,7 +693,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe boolean hasSliderAMLog4j = addAMLog4jResource(appName, conf, localResources); // copy jars to hdfs and add to localResources - Path tempPath = addJarResource(appName, localResources); + addJarResource(appName, localResources); // add keytab if in secure env addKeytabResourceIfSecure(sliderFileSystem, localResources, conf, appName); printLocalResources(localResources); @@ -700,7 +701,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe //TODO SliderAMClientProvider#copyEnvVars //TODO localResource putEnv - Map<String, String> env = addAMEnv(conf, tempPath); + Map<String, String> env = addAMEnv(conf); // create AM CLI String cmdStr = @@ -805,7 +806,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return cmdStr; } - private Map<String, String> addAMEnv(Configuration conf, Path tempPath) + private Map<String, String> addAMEnv(Configuration conf) throws IOException { Map<String, String> env = new HashMap<>(); ClasspathConstructor classpath = @@ -819,6 +820,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe if (jaas != null) { env.put(HADOOP_JAAS_DEBUG, jaas); } + if (!UserGroupInformation.isSecurityEnabled()) { + String userName = UserGroupInformation.getCurrentUser().getUserName(); + log.info("Run as user " + userName); + // HADOOP_USER_NAME env is used by UserGroupInformation when log in + // This env makes AM run as this user + env.put("HADOOP_USER_NAME", userName); + } env.putAll(getAmLaunchEnv(conf)); log.info("AM env: \n{}", stringifyMap(env)); return env; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 9a4fa6c..968a90b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -240,7 +240,7 @@ public interface SliderKeys extends SliderXmlConfKeys { String STDERR_AM = "slider-err.txt"; String DEFAULT_GC_OPTS = ""; - String HADOOP_USER_NAME = ApplicationConstants.Environment.USER.toString(); + String HADOOP_USER_NAME = "HADOOP_USER_NAME"; String HADOOP_PROXY_USER = "HADOOP_PROXY_USER"; String SLIDER_PASSPHRASE = "SLIDER_PASSPHRASE"; @@ -306,7 +306,7 @@ public interface SliderKeys extends SliderXmlConfKeys { String CERT_FILE_LOCALIZATION_PATH = INFRA_RUN_SECURITY_DIR + "ca.crt"; String AM_CONFIG_GENERATION = "am.config.generation"; - String APP_CONF_DIR = "app/conf"; + String APP_CONF_DIR = "conf"; String APP_RESOURCES = "application.resources"; String APP_RESOURCES_DIR = "app/resources"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java index dbaa981..5ab0532 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java @@ -32,7 +32,6 @@ import java.util.List; public class CommandLineBuilder { protected final List<String> argumentList = new ArrayList<>(20); - /** * Add an entry to the command list * @param args arguments -these will be converted strings @@ -44,15 +43,6 @@ public class CommandLineBuilder { } /** - * Get the value at an offset - * @param offset offset - * @return the value at that point - */ - public String elt(int offset) { - return argumentList.get(offset); - } - - /** * Get the number of arguments * @return an integer >= 0 */ @@ -96,9 +86,4 @@ public class CommandLineBuilder { public String build() { return SliderUtils.join(argumentList, " "); } - - public List<String> getArgumentList() { - return argumentList; - } - } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java deleted file mode 100644 index 88bac77..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.registry.docstore; - -public class ConfigurationResolver { - - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java index cb39368..9cc48e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java @@ -20,6 +20,13 @@ package org.apache.slider.providers; import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.resource.Component; +import org.apache.slider.server.appmaster.state.AppState; + +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicLong; /** * Provider role and key for use in app requests. @@ -36,7 +43,9 @@ public final class ProviderRole { public final long placementTimeoutSeconds; public final String labelExpression; public final Component component; - + public AtomicLong componentIdCounter = null; + public AppState appState; + public Queue<String> failedInstanceName = new ConcurrentLinkedQueue<String>(); public ProviderRole(String name, int id) { this(name, id, @@ -69,7 +78,7 @@ public final class ProviderRole { nodeFailureThreshold, placementTimeoutSeconds, labelExpression, - new Component().name(name).numberOfContainers(0L)); + new Component().name(name).numberOfContainers(0L), null); } /** @@ -79,18 +88,13 @@ public final class ProviderRole { * @param id ID. This becomes the YARN priority * @param policy placement policy * @param nodeFailureThreshold threshold for node failures (within a reset interval) - * after which a node failure is considered an app failure +* after which a node failure is considered an app failure * @param placementTimeoutSeconds for lax placement, timeout in seconds before * @param labelExpression label expression for requests; may be null */ - public ProviderRole(String name, - String group, - int id, - int policy, - int nodeFailureThreshold, - long placementTimeoutSeconds, - String labelExpression, - Component component) { + public ProviderRole(String name, String group, int id, int policy, + int nodeFailureThreshold, long placementTimeoutSeconds, + String labelExpression, Component component, AppState state) { this.name = name; if (group == null) { this.group = name; @@ -103,9 +107,13 @@ public final class ProviderRole { this.placementTimeoutSeconds = placementTimeoutSeconds; this.labelExpression = labelExpression; this.component = component; - + if(component.getUniqueComponentSupport()) { + componentIdCounter = new AtomicLong(0); + } + this.appState = state; } + @Override public boolean equals(Object o) { if (this == o) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java index c31b2ac..7e92bfa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java @@ -26,6 +26,7 @@ import org.apache.slider.api.resource.Application; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.ContainerLauncher; +import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; @@ -38,7 +39,8 @@ public interface ProviderService extends Service { */ void buildContainerLaunchContext(ContainerLauncher containerLauncher, Application application, Container container, ProviderRole providerRole, - SliderFileSystem sliderFileSystem) throws IOException, SliderException; + SliderFileSystem sliderFileSystem, RoleInstance roleInstance) + throws IOException, SliderException; void setAMState(StateAccessForProviders stateAccessForProviders); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java index f8ec976..d384585 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java @@ -20,10 +20,10 @@ package org.apache.slider.providers; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; @@ -32,7 +32,6 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.slider.api.ClusterNode; -import org.apache.slider.api.InternalKeys; import org.apache.slider.api.OptionKeys; import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.RoleKeys; @@ -59,6 +58,7 @@ import org.slf4j.Logger; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.OutputStream; import java.net.URI; import java.util.ArrayList; import java.util.Collection; @@ -66,8 +66,12 @@ import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.regex.Pattern; +import static org.apache.slider.api.ServiceApiConstants.*; +import static org.apache.slider.util.ServiceApiUtil.$; + /** * This is a factoring out of methods handy for providers. It's bonded to a log * at construction time. @@ -89,7 +93,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys { * Add oneself to the classpath. This does not work * on minicluster test runs where the JAR is not built up. * @param providerResources map of provider resources to add these entries to - * @param provider provider to add + * @param providerClass provider to add * @param jarName name of the jar to use * @param sliderFileSystem target filesystem * @param tempPath path in the cluster FS for temp files @@ -157,14 +161,19 @@ public class ProviderUtils implements RoleKeys, SliderKeys { libDir, libLocalSrcDir); } - // Build key -> value map - // value will be substituted by corresponding data in tokenMap - public Map<String, String> substituteConfigs(Map<String, String> configs, + public static String substituteStrWithTokens(String content, + Map<String, String> tokensForSubstitution) { + for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) { + content = + content.replaceAll(Pattern.quote(token.getKey()), token.getValue()); + } + return content; + } + + // configs will be substituted by corresponding env in tokenMap + public void substituteMapWithTokens(Map<String, String> configs, Map<String, String> tokenMap) { - String format = "${%s}"; - Map<String, String> filteredOptions = new HashMap<>(); for (Map.Entry<String, String> entry : configs.entrySet()) { - String key = entry.getKey(); String value = entry.getValue(); if (tokenMap != null) { for (Map.Entry<String, String> token : tokenMap.entrySet()) { @@ -172,10 +181,8 @@ public class ProviderUtils implements RoleKeys, SliderKeys { value.replaceAll(Pattern.quote(token.getKey()), token.getValue()); } } - filteredOptions.put(String.format(format, key), value); + entry.setValue(value); } - - return filteredOptions; } /** @@ -249,78 +256,95 @@ public class ProviderUtils implements RoleKeys, SliderKeys { } } + public static void addEnvForSubstitution(Map<String, String> env, + Map<String, String> tokensForSubstitution) { + if (env == null || env.isEmpty() || tokensForSubstitution == null + || tokensForSubstitution.isEmpty()) { + return; + } + for (Map.Entry<String, String> entry : env.entrySet()) { + tokensForSubstitution.put($(entry.getKey()), entry.getValue()); + } + } // 1. Create all config files for a component on hdfs for localization // 2. Add the config file to localResource - //TODO handle Template format config file - public void createConfigFileAndAddLocalResource(ContainerLauncher launcher, - SliderFileSystem fs, Component component, - Map<String, String> tokensForSubstitution, - StateAccessForProviders amState) throws IOException { + public synchronized void createConfigFileAndAddLocalResource( + ContainerLauncher launcher, SliderFileSystem fs, Component component, + Map<String, String> tokensForSubstitution, RoleInstance roleInstance) + throws IOException { Path compDir = new Path(new Path(fs.getAppDir(), "components"), component.getName()); - if (!fs.getFileSystem().exists(compDir)) { - fs.getFileSystem().mkdirs(compDir, + Path compInstanceDir = + new Path(compDir, roleInstance.getCompInstanceName()); + if (!fs.getFileSystem().exists(compInstanceDir)) { + fs.getFileSystem().mkdirs(compInstanceDir, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); - log.info("Creating component dir: " + compDir); + roleInstance.compInstanceDir = compInstanceDir; + log.info("Creating component instance dir: " + compInstanceDir); } else { - log.info("Component conf dir already exists: " + compDir); - return; + log.info("Component instance conf dir already exists: " + compInstanceDir); } - for (ConfigFile configFile : component.getConfiguration().getFiles()) { - String fileName = configFile.getSrcFile(); + // add Configuration#env into tokens substitution + addEnvForSubstitution(component.getConfiguration().getEnv(), + tokensForSubstitution); + + log.info("Tokens substitution for component: " + roleInstance + .getCompInstanceName() + System.lineSeparator() + + tokensForSubstitution); + + for (ConfigFile originalFile : component.getConfiguration().getFiles()) { + ConfigFile configFile = originalFile.copy(); + String fileName = new Path(configFile.getDestFile()).getName(); + // substitute file name for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) { configFile.setDestFile(configFile.getDestFile() .replaceAll(Pattern.quote(token.getKey()), token.getValue())); } - // substitute configs - substituteConfigs(configFile.getProps(), tokensForSubstitution); - - // write configs onto hdfs - PublishedConfiguration publishedConfiguration = - new PublishedConfiguration(fileName, - configFile.getProps().entrySet()); - Path remoteFile = new Path(compDir, fileName); + + Path remoteFile = new Path(compInstanceDir, fileName); if (!fs.getFileSystem().exists(remoteFile)) { - synchronized (this) { - if (!fs.getFileSystem().exists(remoteFile)) { - PublishedConfigurationOutputter configurationOutputter = - PublishedConfigurationOutputter.createOutputter( - ConfigFormat.resolve(configFile.getType().toString()), - publishedConfiguration); - FSDataOutputStream os = null; - try { - os = fs.getFileSystem().create(remoteFile); - configurationOutputter.save(os); - os.flush(); - log.info("Created config file on hdfs: " + remoteFile); - } finally { - IOUtils.closeStream(os); - } + log.info("Saving config file on hdfs for component " + roleInstance + .getCompInstanceName() + ": " + configFile); + + if (configFile.getSrcFile() != null) { + // Load config file template + switch (configFile.getType()) { + case HADOOP_XML: + // Hadoop_xml_template + resolveHadoopXmlTemplateAndSaveOnHdfs(fs.getFileSystem(), + tokensForSubstitution, configFile, remoteFile, roleInstance); + break; + case TEMPLATE: + // plain-template + resolvePlainTemplateAndSaveOnHdfs(fs.getFileSystem(), + tokensForSubstitution, configFile, remoteFile, roleInstance); + break; + default: + log.info("Not supporting loading src_file for " + configFile); + break; } + } else { + // non-template + resolveNonTemplateConfigsAndSaveOnHdfs(fs, tokensForSubstitution, + roleInstance, configFile, fileName, remoteFile); } } - // Publish configs - amState.getPublishedSliderConfigurations() - .put(configFile.getSrcFile(), publishedConfiguration); - // Add resource for localization LocalResource configResource = fs.createAmResource(remoteFile, LocalResourceType.FILE); File destFile = new File(configFile.getDestFile()); - //TODO why to we need to differetiate RESOURCE_DIR vs APP_CONF_DIR + String symlink = APP_CONF_DIR + "/" + fileName; if (destFile.isAbsolute()) { - String symlink = RESOURCE_DIR + "/" + fileName; launcher.addLocalResource(symlink, configResource, configFile.getDestFile()); log.info("Add config file for localization: " + symlink + " -> " + configResource.getResource().getFile() + ", dest mount path: " + configFile.getDestFile()); } else { - String symlink = APP_CONF_DIR + "/" + fileName; launcher.addLocalResource(symlink, configResource); log.info("Add config file for localization: " + symlink + " -> " + configResource.getResource().getFile()); @@ -328,23 +352,110 @@ public class ProviderUtils implements RoleKeys, SliderKeys { } } + private void resolveNonTemplateConfigsAndSaveOnHdfs(SliderFileSystem fs, + Map<String, String> tokensForSubstitution, RoleInstance roleInstance, + ConfigFile configFile, String fileName, Path remoteFile) + throws IOException { + // substitute non-template configs + substituteMapWithTokens(configFile.getProps(), tokensForSubstitution); + + // write configs onto hdfs + PublishedConfiguration publishedConfiguration = + new PublishedConfiguration(fileName, + configFile.getProps().entrySet()); + if (!fs.getFileSystem().exists(remoteFile)) { + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter( + ConfigFormat.resolve(configFile.getType().toString()), + publishedConfiguration); + try (FSDataOutputStream os = fs.getFileSystem().create(remoteFile)) { + configurationOutputter.save(os); + os.flush(); + } + } else { + log.info("Component instance = " + roleInstance.getCompInstanceName() + + ", config file already exists: " + remoteFile); + } + } + + // 1. substitute config template - only handle hadoop_xml format + // 2. save on hdfs + @SuppressWarnings("unchecked") + private void resolveHadoopXmlTemplateAndSaveOnHdfs(FileSystem fs, + Map<String, String> tokensForSubstitution, ConfigFile configFile, + Path remoteFile, RoleInstance roleInstance) throws IOException { + Map<String, String> conf; + try { + conf = (Map<String, String>) roleInstance.providerRole. + appState.configFileCache.get(configFile); + } catch (ExecutionException e) { + log.info("Failed to load config file: " + configFile, e); + return; + } + // make a copy for substitution + org.apache.hadoop.conf.Configuration confCopy = + new org.apache.hadoop.conf.Configuration(false); + for (Map.Entry<String, String> entry : conf.entrySet()) { + confCopy.set(entry.getKey(), entry.getValue()); + } + // substitute properties + for (Map.Entry<String, String> entry : configFile.getProps().entrySet()) { + confCopy.set(entry.getKey(), entry.getValue()); + } + // substitute env variables + for (Map.Entry<String, String> entry : confCopy) { + String val = entry.getValue(); + if (val != null) { + for (Map.Entry<String, String> token : tokensForSubstitution + .entrySet()) { + val = val.replaceAll(Pattern.quote(token.getKey()), token.getValue()); + confCopy.set(entry.getKey(), val); + } + } + } + // save on hdfs + try (OutputStream output = fs.create(remoteFile)) { + confCopy.writeXml(output); + log.info("Reading config from: " + configFile.getSrcFile() + + ", writing to: " + remoteFile); + } + } + + // 1) read the template as a string + // 2) do token substitution + // 3) save on hdfs + private void resolvePlainTemplateAndSaveOnHdfs(FileSystem fs, + Map<String, String> tokensForSubstitution, ConfigFile configFile, + Path remoteFile, RoleInstance roleInstance) { + String content; + try { + content = (String) roleInstance.providerRole.appState.configFileCache + .get(configFile); + } catch (ExecutionException e) { + log.info("Failed to load config file: " + configFile, e); + return; + } + // substitute tokens + substituteStrWithTokens(content, tokensForSubstitution); + + try (OutputStream output = fs.create(remoteFile)) { + org.apache.commons.io.IOUtils.write(content, output); + } catch (IOException e) { + log.info("Failed to create " + remoteFile); + } + } + /** * Get initial token map to be substituted into config values. * @param appConf app configurations - * @param componentName component name - * @param componentGroup component group - * @param containerId container ID * @param clusterName app name * @return tokens to replace */ - public Map<String, String> getStandardTokenMap( - Configuration appConf, Configuration componentConf, String componentName, - String componentGroup, String containerId, String clusterName) { + public Map<String, String> getStandardTokenMap(Configuration appConf, + RoleInstance roleInstance, String clusterName) { Map<String, String> tokens = new HashMap<>(); - if (containerId != null) { - tokens.put("${CONTAINER_ID}", containerId); - } + String nnuri = appConf.getProperty("fs.defaultFS"); if (nnuri != null && !nnuri.isEmpty()) { tokens.put("${NN_URI}", nnuri); @@ -352,34 +463,13 @@ public class ProviderUtils implements RoleKeys, SliderKeys { } tokens.put("${ZK_HOST}", appConf.getProperty(OptionKeys.ZOOKEEPER_HOSTS)); tokens.put("${DEFAULT_ZK_PATH}", appConf.getProperty(OptionKeys.ZOOKEEPER_PATH)); - String prefix = componentConf.getProperty(ROLE_PREFIX); - String dataDirSuffix = ""; - if (prefix == null) { - prefix = ""; - } else { - dataDirSuffix = "_" + SliderUtils.trimPrefix(prefix); - } - tokens.put("${DEFAULT_DATA_DIR}", - appConf.getProperty(InternalKeys.INTERNAL_DATA_DIR_PATH) - + dataDirSuffix); - tokens.put("${JAVA_HOME}", appConf.getProperty(JAVA_HOME)); - tokens.put("${COMPONENT_NAME}", componentName); - tokens.put("${COMPONENT_NAME.lc}", componentName.toLowerCase()); - tokens.put("${COMPONENT_PREFIX}", prefix); - tokens.put("${COMPONENT_PREFIX.lc}", prefix.toLowerCase()); - if (!componentName.equals(componentGroup) && - componentName.startsWith(componentGroup)) { - tokens.put("${COMPONENT_ID}", - componentName.substring(componentGroup.length())); - } - if (clusterName != null) { - tokens.put("${CLUSTER_NAME}", clusterName); - tokens.put("${CLUSTER_NAME.lc}", clusterName.toLowerCase()); - tokens.put("${APP_NAME}", clusterName); - tokens.put("${APP_NAME.lc}", clusterName.toLowerCase()); - } - tokens.put("${APP_COMPONENT_NAME}", componentName); - tokens.put("${APP_COMPONENT_NAME.lc}", componentName.toLowerCase()); + tokens.put(SERVICE_NAME_LC, clusterName.toLowerCase()); + tokens.put(SERVICE_NAME, clusterName); + tokens.put(COMPONENT_NAME, roleInstance.role); + tokens.put(COMPONENT_NAME_LC, roleInstance.role.toLowerCase()); + tokens.put(COMPONENT_INSTANCE_NAME, roleInstance.getCompInstanceName()); + tokens.put(CONTAINER_ID, roleInstance.getContainerId().toString()); + tokens.put(COMPONENT_ID, String.valueOf(roleInstance.componentId)); return tokens; } @@ -388,7 +478,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys { * @param tokens existing tokens * @param amState access to AM state */ - public void addRoleHostTokens(Map<String, String> tokens, + public void addComponentHostTokens(Map<String, String> tokens, StateAccessForProviders amState) { if (amState == null) { return; @@ -398,7 +488,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys { String tokenName = entry.getKey().toUpperCase(Locale.ENGLISH) + "_HOST"; String hosts = StringUtils .join(",", getHostsList(entry.getValue().values(), true)); - tokens.put("${" + tokenName + "}", hosts); + tokens.put($(tokenName), hosts); } } @@ -443,7 +533,8 @@ public class ProviderUtils implements RoleKeys, SliderKeys { // create and publish updated service record (including hostname & ip) ServiceRecord record = new ServiceRecord(); record.set(YarnRegistryAttributes.YARN_ID, containerId); - record.description = roleName.replaceAll("_", "-"); + String componentInstanceName = role.getCompInstanceName(); + record.description = componentInstanceName.replaceAll("_", "-"); record.set(YarnRegistryAttributes.YARN_PERSISTENCE, PersistencePolicies.CONTAINER); // TODO: use constants from YarnRegistryAttributes http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java index 511f7bc..93a481c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java @@ -46,6 +46,8 @@ import java.io.IOException; import java.util.Map; import java.util.Map.Entry; +import static org.apache.slider.util.ServiceApiUtil.$; + public class DockerProviderService extends AbstractService implements ProviderService, DockerKeys, SliderKeys { @@ -70,14 +72,11 @@ public class DockerProviderService extends AbstractService this.yarnRegistry = yarnRegistry; } + public void buildContainerLaunchContext(ContainerLauncher launcher, Application application, Container container, ProviderRole providerRole, - SliderFileSystem fileSystem) + SliderFileSystem fileSystem, RoleInstance roleInstance) throws IOException, SliderException { - - String roleName = providerRole.name; - String roleGroup = providerRole.group; - Component component = providerRole.component; launcher.setYarnDockerMode(true); launcher.setDockerImage(component.getArtifact().getId()); @@ -86,16 +85,12 @@ public class DockerProviderService extends AbstractService launcher.setRunPrivilegedContainer(component.getRunPrivilegedContainer()); // Generate tokens (key-value pair) for config substitution. - Map<String, String> standardTokens = providerUtils - .getStandardTokenMap(application.getConfiguration(), - component.getConfiguration(), roleName, roleGroup, - container.getId().toString(), application.getName()); - Map<String, String> tokensForSubstitution = providerUtils.substituteConfigs( - component.getConfiguration().getProperties(), standardTokens); - - tokensForSubstitution.putAll(standardTokens); + // Get pre-defined tokens + Map<String, String> tokensForSubstitution = providerUtils + .getStandardTokenMap(application.getConfiguration(), roleInstance, + application.getName()); - // Set the environment variables + // Set the environment variables in launcher launcher.putEnv(SliderUtils .buildEnvMap(component.getConfiguration(), tokensForSubstitution)); launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$()); @@ -108,33 +103,26 @@ public class DockerProviderService extends AbstractService launcher.setEnv("LANGUAGE", "en_US.UTF-8"); for (Entry<String, String> entry : launcher.getEnv().entrySet()) { - tokensForSubstitution.put("${" + entry.getKey() + "}", entry.getValue()); - } - - providerUtils.addRoleHostTokens(tokensForSubstitution, amState); - - log.info("Token for substitution: " + tokensForSubstitution); - - if (SliderUtils.isHadoopClusterSecure(getConfig())) { - //TODO localize key tabs, WHY is this code needed ? WHY DOES CONTAINER REQUIRE AM KEYTAB?? - providerUtils.localizeServiceKeytabs(launcher, fileSystem, application); + tokensForSubstitution.put($(entry.getKey()), entry.getValue()); } + providerUtils.addComponentHostTokens(tokensForSubstitution, amState); // create config file on hdfs and add local resource providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem, - component, tokensForSubstitution, amState); + component, tokensForSubstitution, roleInstance); + // substitute launch command + String launchCommand = ProviderUtils + .substituteStrWithTokens(component.getLaunchCommand(), + tokensForSubstitution); CommandLineBuilder operation = new CommandLineBuilder(); - operation.add(component.getLaunchCommand()); - operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" - + OUT_FILE + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" - + ERR_FILE); + operation.add(launchCommand); + operation.addOutAndErrFiles(OUT_FILE, ERR_FILE); launcher.addCommand(operation.build()); // publish exports - // TODO move this to app level, no need to do this for every container launch providerUtils - .substituteConfigs(application.getQuicklinks(), tokensForSubstitution); + .substituteMapWithTokens(application.getQuicklinks(), tokensForSubstitution); PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS, application.getQuicklinks().entrySet()); amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java index 70eab71..c53349f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java @@ -150,24 +150,26 @@ public class RoleLaunchService containerLauncher.setupUGI(); containerLauncher.putEnv(envVars); - log.info("Launching container {} into RoleName = {}, RoleGroup = {}", - container.getId(), role.name, role.group); - - provider.buildContainerLaunchContext(containerLauncher, application, - container, role, fs); - - RoleInstance instance = new RoleInstance(container); + String failedInstance = role.failedInstanceName.poll(); + RoleInstance instance; + if (failedInstance != null) { + instance = new RoleInstance(container, role, failedInstance); + } else { + instance = new RoleInstance(container, role); + } String[] envDescription = containerLauncher.dumpEnvToString(); - String commandsAsString = containerLauncher.getCommandsAsString(); + log.info("Launching container {} for component instance = {}", + container.getId(), instance.getCompInstanceName()); log.info("Starting container with command: {}", commandsAsString); - - instance.providerRole = role; instance.command = commandsAsString; instance.role = role.name; - instance.group = role.group; instance.roleId = role.id; instance.environment = envDescription; + + provider.buildContainerLaunchContext(containerLauncher, application, + container, role, fs, instance); + long delay = role.component.getConfiguration() .getPropertyLong(AgentKeys.KEY_CONTAINER_LAUNCH_DELAY, 0); long maxDelay = getConfig() http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 7473dab..1f379ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -25,6 +25,7 @@ import com.google.protobuf.BlockingService; import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; @@ -86,6 +87,7 @@ import org.apache.slider.api.RoleKeys; import org.apache.slider.api.proto.Messages; import org.apache.slider.api.proto.SliderClusterAPI; import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.params.AbstractActionArgs; @@ -109,7 +111,6 @@ import org.apache.slider.core.main.ServiceLauncher; import org.apache.slider.core.persist.JsonSerDeser; import org.apache.slider.core.registry.info.CustomRegistryConstants; import org.apache.slider.providers.ProviderCompleted; -import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderService; import org.apache.slider.providers.SliderProviderFactory; import org.apache.slider.server.appmaster.actions.ActionHalt; @@ -136,7 +137,6 @@ import org.apache.slider.server.appmaster.operations.RMOperationHandler; import org.apache.slider.server.appmaster.rpc.RpcBinder; import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPBImpl; import org.apache.slider.server.appmaster.rpc.SliderIPCService; -import org.apache.slider.server.appmaster.security.SecurityConfiguration; import org.apache.slider.server.appmaster.state.AppState; import org.apache.slider.server.appmaster.state.AppStateBindingInfo; import org.apache.slider.server.appmaster.state.ContainerAssignment; @@ -170,7 +170,6 @@ import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -701,10 +700,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService registryOperations = startRegistryOperationsService(); log.info(registryOperations.toString()); - //build the role map - List<ProviderRole> providerRoles = Collections.EMPTY_LIST; // Start up the WebApp and track the URL for it - // Web service endpoints: initialize WebAppApiImpl webAppApi = new WebAppApiImpl( @@ -815,7 +811,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService //build the instance AppStateBindingInfo binding = new AppStateBindingInfo(); binding.serviceConfig = serviceConf; - binding.roles = providerRoles; binding.fs = fs.getFileSystem(); binding.historyPath = historyDir; binding.liveContainers = liveContainers; @@ -873,6 +868,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService scheduleFailureWindowResets(application.getConfiguration()); scheduleEscalation(application.getConfiguration()); + for (Component component : application.getComponents()) { + // Merge app-level configuration into component level configuration + component.getConfiguration().mergeFrom(application.getConfiguration()); + } + try { // schedule YARN Registry registration queue(new ActionRegisterServiceInstance(appName, appid, application)); @@ -1170,22 +1170,22 @@ public class SliderAppMaster extends AbstractSliderLaunchedService * Handler for {@link RegisterComponentInstance action} * Register/re-register an ephemeral container that is already in the application state * @param id the component - * @param description component description - * @param type component type * @return true if the component is registered */ - public boolean registerComponent(ContainerId id, String description, - String type) throws IOException { + public boolean registerComponent(ContainerId id, RoleInstance roleInstance) + throws IOException { RoleInstance instance = appState.getOwnedContainer(id); if (instance == null) { return false; } // this is where component registrations go - log.info("Registering component {}", id); String cid = RegistryPathUtils.encodeYarnID(id.toString()); ServiceRecord record = new ServiceRecord(); record.set(YarnRegistryAttributes.YARN_ID, cid); - record.description = description; + + record.description = roleInstance.getCompInstanceName(); + log.info("Registering component " + roleInstance.getCompInstanceName() + + ", containerId = " + id); record.set(YarnRegistryAttributes.YARN_PERSISTENCE, PersistencePolicies.CONTAINER); setUserProvidedServiceRecordAttributes( @@ -1194,7 +1194,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService yarnRegistryOperations.putComponent(cid, record); } catch (IOException e) { log.warn("Failed to register container {}/{}: {}", - id, description, e, e); + id, roleInstance.role, e, e); return false; } org.apache.slider.api.resource.Container container = @@ -1203,6 +1203,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService container.setLaunchTime(new Date()); container.setState(org.apache.slider.api.resource.ContainerState.INIT); container.setBareHost(instance.host); + // TODO differentiate component name and component instance name ? + container.setComponentName(roleInstance.getCompInstanceName()); instance.providerRole.component.addContainer(container); if (timelineServiceEnabled) { @@ -1228,20 +1230,38 @@ public class SliderAppMaster extends AbstractSliderLaunchedService * * unregister a component. At the time this message is received, * the component may not have been registered - * @param id the component */ - public void unregisterComponent(ContainerId id) { - log.info("Unregistering component {}", id); + public void unregisterComponent(RoleInstance roleInstance) { + ContainerId containerId = roleInstance.getContainerId(); + log.info( + "Unregistering component instance " + roleInstance.getCompInstanceName() + + ", ContainerId = " + containerId); if (yarnRegistryOperations == null) { - log.warn("Processing unregister component event before initialization " + - "completed; init flag ={}", initCompleted); + log.warn("Processing unregister component event before initialization " + + "completed; init flag ={}", initCompleted); return; } - String cid = RegistryPathUtils.encodeYarnID(id.toString()); + String cid = RegistryPathUtils.encodeYarnID(containerId.toString()); try { yarnRegistryOperations.deleteComponent(cid); } catch (IOException e) { - log.warn("Failed to delete container {} : {}", id, e, e); + log.warn("Failed to delete container {} : {}", containerId, e, e); + } + + // remove component instance dir + try { + FileSystem fs = getClusterFS().getFileSystem(); + if (roleInstance.compInstanceDir != null && fs + .exists(roleInstance.compInstanceDir)) { + boolean deleted = fs.delete(roleInstance.compInstanceDir, true); + if (!deleted) { + log.warn("Failed to delete component instance dir: " + + roleInstance.compInstanceDir); + } + } + } catch (IOException e) { + log.error("Failed to delete component instance dir: " + + roleInstance.compInstanceDir, e); } } @@ -1395,13 +1415,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService return exitCode; } - /** - * Get diagnostics info about containers - */ - private String getContainerDiagnosticInfo() { - - return appState.getContainerDiagnosticInfo(); - } public Object getProxy(Class protocol, InetSocketAddress addr) { return yarnRPC.getProxy(protocol, addr, getConfig()); @@ -1492,7 +1505,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService //for all the operations, exec them execute(operations); - log.info("Diagnostics: {}", getContainerDiagnosticInfo()); } @Override //AMRMClientAsync @@ -1519,8 +1531,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // known nodes trigger notifications if(!result.unknownNode) { - queue(new UnregisterComponentInstance(containerId, 0, - TimeUnit.MILLISECONDS)); + queue(new UnregisterComponentInstance(0, + TimeUnit.MILLISECONDS, result.roleInstance)); + if (timelineServiceEnabled && result.roleInstance != null) { serviceTimelinePublisher .componentInstanceFinished(result.roleInstance); @@ -1936,7 +1949,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService nmClientAsync.getContainerStatusAsync(containerId, cinfo.container.getNodeId()); // push out a registration - queue(new RegisterComponentInstance(containerId, cinfo.role, cinfo.group, + queue(new RegisterComponentInstance(containerId, cinfo, 0, TimeUnit.MILLISECONDS)); } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java index 4cf4981..3c1bed8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.slider.server.appmaster.SliderAppMaster; import org.apache.slider.server.appmaster.state.AppState; +import org.apache.slider.server.appmaster.state.RoleInstance; import java.util.concurrent.TimeUnit; @@ -33,18 +34,15 @@ import java.util.concurrent.TimeUnit; public class RegisterComponentInstance extends AsyncAction { public final ContainerId containerId; - public final String description; - public final String type; + public final RoleInstance roleInstance; public RegisterComponentInstance(ContainerId containerId, - String description, - String type, + RoleInstance roleInstance, long delay, TimeUnit timeUnit) { super("RegisterComponentInstance :" + containerId, delay, timeUnit); - this.description = description; - this.type = type; + this.roleInstance = roleInstance; Preconditions.checkArgument(containerId != null); this.containerId = containerId; } @@ -54,6 +52,6 @@ public class RegisterComponentInstance extends AsyncAction { QueueAccess queueService, AppState appState) throws Exception { - appMaster.registerComponent(containerId, description, type); + appMaster.registerComponent(containerId, roleInstance); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java index 575fe8f..ac86333 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java @@ -21,31 +21,31 @@ package org.apache.slider.server.appmaster.actions; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.slider.server.appmaster.SliderAppMaster; import org.apache.slider.server.appmaster.state.AppState; +import org.apache.slider.server.appmaster.state.RoleInstance; import java.util.concurrent.TimeUnit; /** * Tell AM to unregister this component instance - * {@link SliderAppMaster#unregisterComponent(ContainerId)} */ public class UnregisterComponentInstance extends AsyncAction { - public final ContainerId containerId; + public final RoleInstance roleInstance; - public UnregisterComponentInstance(ContainerId containerId, - long delay, - TimeUnit timeUnit) { - super("UnregisterComponentInstance :" + containerId.toString(), + public UnregisterComponentInstance(long delay, TimeUnit timeUnit, + RoleInstance roleInstance) { + super("UnregisterComponentInstance :" + roleInstance.getCompInstanceName() + + ", ContainerId = " + roleInstance.getContainerId(), delay, timeUnit); - this.containerId = containerId; + this.roleInstance = roleInstance; } @Override public void execute(SliderAppMaster appMaster, QueueAccess queueService, AppState appState) throws Exception { - appMaster.unregisterComponent(containerId); + appMaster.unregisterComponent(roleInstance); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java index 5dcbe9b..cf607a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java @@ -39,24 +39,34 @@ public class SliderMetrics implements MetricsSource { @Metric("containers requested") public MutableGaugeInt containersRequested; + @Metric("anti-affinity containers pending") public MutableGaugeInt pendingAAContainers; + @Metric("containers running") public MutableGaugeInt containersRunning; + @Metric("containers desired") public MutableGaugeInt containersDesired; + @Metric("containers completed") public MutableGaugeInt containersCompleted; + @Metric("containers failed") public MutableGaugeInt containersFailed; + @Metric("containers failed since last threshold") public MutableGaugeInt failedSinceLastThreshold; + @Metric("containers preempted") public MutableGaugeInt containersPreempted; + @Metric("containers exceeded limits") public MutableGaugeInt containersLimitsExceeded; + @Metric("containers surplus") public MutableGaugeInt surplusContainers; + @Metric("containers failed due to disk failure") public MutableGaugeInt containersDiskFailure; @@ -80,5 +90,18 @@ public class SliderMetrics implements MetricsSource { public void tag(String name, String description, String value) { registry.tag(name, description, value); } + + @Override public String toString() { + return "SliderMetrics{" + + "containersRequested=" + containersRequested.value() + + ", pendingAAContainers=" + pendingAAContainers.value() + + ", containersRunning=" + containersRunning.value() + + ", containersDesired=" + containersDesired.value() + + ", containersCompleted=" + containersCompleted.value() + + ", containersFailed=" + containersFailed.value() + + ", failedSinceLastThreshold=" + failedSinceLastThreshold.value() + + ", containersPreempted=" + containersPreempted.value() + + ", surplusContainers=" + surplusContainers.value() + '}'; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java index ae38e4c..1406fbe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java @@ -77,7 +77,7 @@ public class ChaosKillContainer implements ChaosTarget { RoleInstance roleInstance = liveContainers.get(target); log.info("Killing {}", roleInstance); - queues.schedule(new ActionKillContainer(roleInstance.getId(), + queues.schedule(new ActionKillContainer(roleInstance.getContainerId(), DELAY, TimeUnit.MILLISECONDS, operationHandler)); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d23a97d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java index f88d586..22f9bc3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java @@ -283,7 +283,7 @@ public class SliderIPCService extends AbstractService //throws NoSuchNodeException if it is missing RoleInstance instance = state.getLiveInstanceByContainerID(containerID); - queue(new ActionKillContainer(instance.getId(), 0, TimeUnit.MILLISECONDS, + queue(new ActionKillContainer(instance.getContainerId(), 0, TimeUnit.MILLISECONDS, amOperations)); Messages.KillContainerResponseProto.Builder builder = Messages.KillContainerResponseProto.newBuilder(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org