http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java index 7bfb410..abad34e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java @@ -17,16 +17,15 @@ package org.apache.hadoop.yarn.services.api.impl; -import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*; -import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*; +import static org.apache.slider.util.RestApiConstants.*; +import static org.apache.slider.util.RestApiErrorMessages.*; -import java.util.HashMap; -import java.util.Map; +import java.util.ArrayList; import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.Artifact; import org.apache.slider.api.resource.Resource; -import org.apache.slider.common.SliderKeys; +import org.apache.slider.util.ServiceApiUtil; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -61,12 +60,10 @@ public class TestApplicationApiService { @Test(timeout = 90000) public void testValidateApplicationPostPayload() throws Exception { Application app = new Application(); - Map<String, String> compNameArtifactIdMap = new HashMap<>(); // no name try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with no name"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage()); @@ -77,8 +74,7 @@ public class TestApplicationApiService { for (String badName : badNames) { app.setName(badName); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID_FORMAT, @@ -89,8 +85,7 @@ public class TestApplicationApiService { // no artifact app.setName("finance_home"); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with no artifact"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_ARTIFACT_INVALID, e.getMessage()); @@ -100,8 +95,7 @@ public class TestApplicationApiService { Artifact artifact = new Artifact(); app.setArtifact(artifact); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with no artifact id"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage()); @@ -112,8 +106,7 @@ public class TestApplicationApiService { artifact.setId("app.io/hbase:facebook_0.2"); app.setNumberOfContainers(5l); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); } catch (IllegalArgumentException e) { logger.error("application attributes specified should be valid here", e); Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); @@ -124,22 +117,18 @@ public class TestApplicationApiService { Assert.assertEquals(app.getComponents().get(0).getName(), DEFAULT_COMPONENT_NAME); Assert.assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME); - Assert.assertEquals("Property not set", - app.getConfiguration().getProperties() - .get(SliderKeys.COMPONENT_TYPE_KEY), - SliderKeys.COMPONENT_TYPE_EXTERNAL_APP); + //TODO handle external app // unset artifact type, default component and no of containers to test other // validation logic artifact.setType(null); - app.setComponents(null); + app.setComponents(new ArrayList<>()); app.setNumberOfContainers(null); // resource not specified artifact.setId("docker.io/centos:centos7"); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with no resource"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_RESOURCE_INVALID, e.getMessage()); @@ -149,28 +138,18 @@ public class TestApplicationApiService { Resource res = new Resource(); app.setResource(res); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with no memory"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_RESOURCE_MEMORY_INVALID, e.getMessage()); } - // cpus not specified - res.setMemory("2gb"); - try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); - Assert.fail(EXCEPTION_PREFIX + "application with no cpu"); - } catch (IllegalArgumentException e) { - Assert.assertEquals(ERROR_RESOURCE_CPUS_INVALID, e.getMessage()); - } - + // cpu does not need to be always specified, it's an optional feature in yarn // invalid no of cpus + res.setMemory("100mb"); res.setCpus(-2); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail( EXCEPTION_PREFIX + "application with invalid no of cpups"); } catch (IllegalArgumentException e) { @@ -180,8 +159,7 @@ public class TestApplicationApiService { // number of containers not specified res.setCpus(2); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail( EXCEPTION_PREFIX + "application with no container count"); } catch (IllegalArgumentException e) { @@ -191,8 +169,7 @@ public class TestApplicationApiService { // specifying profile along with cpus/memory raises exception res.setProfile("hbase_finance_large"); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with resource profile along with cpus/memory"); } catch (IllegalArgumentException e) { @@ -205,8 +182,7 @@ public class TestApplicationApiService { res.setCpus(null); res.setMemory(null); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with resource profile only - NOT SUPPORTED"); } catch (IllegalArgumentException e) { @@ -222,8 +198,7 @@ public class TestApplicationApiService { // everything valid here app.setNumberOfContainers(5l); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); } catch (IllegalArgumentException e) { logger.error("application attributes specified should be valid here", e); Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java index d21785f..f6a2cc9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java @@ -60,14 +60,6 @@ public interface SliderApplicationApi { ConfTreeOperations getDesiredResources() throws IOException; /** - * Put an updated resources structure. This triggers a cluster flex - * operation - * @param updated updated resources - * @throws IOException on any problem. - */ - void putDesiredResources(ConfTree updated) throws IOException; - - /** * Get the aggregate resolved model * @return the aggregate configuration of what was asked for * -after resolution has taken place http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java index 893e706..f384927 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java @@ -52,12 +52,9 @@ public interface SliderClusterProtocol extends VersionedProtocol { Messages.UpgradeContainersRequestProto request) throws IOException, YarnException; - /** - * Flex the cluster. - */ - Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request) - throws IOException; + Messages.FlexComponentResponseProto flexComponent( + Messages.FlexComponentRequestProto request) throws IOException; /** * Get the current cluster status @@ -121,13 +118,6 @@ public interface SliderClusterProtocol extends VersionedProtocol { throws IOException; /** - * Get the instance definition - */ - Messages.GetInstanceDefinitionResponseProto getInstanceDefinition( - Messages.GetInstanceDefinitionRequestProto request) - throws IOException, YarnException; - - /** * Get the application liveness * @return current liveness information * @throws IOException http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java index cc3355a..502b519 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java @@ -28,6 +28,7 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -55,11 +56,11 @@ public class Application extends BaseResource { private Long numberOfRunningContainers = null; private Long lifetime = null; private PlacementPolicy placementPolicy = null; - private List<Component> components = null; - private Configuration configuration = null; + private List<Component> components = new ArrayList<>(); + private Configuration configuration = new Configuration(); private List<Container> containers = new ArrayList<>(); private ApplicationState state = null; - private Map<String, String> quicklinks = null; + private Map<String, String> quicklinks = new HashMap<>(); private String queue = null; /** @@ -285,6 +286,15 @@ public class Application extends BaseResource { this.components = components; } + public Component getComponent(String name) { + for (Component component : components) { + if (component.getName().equals(name)) { + return component; + } + } + return null; + } + /** * Config properties of an application. Configurations provided at the * application/global level are available to all the components. Specific http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java index 4f50564..e7f3796 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java @@ -22,6 +22,7 @@ import io.swagger.annotations.ApiModelProperty; import java.io.Serializable; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; @@ -49,15 +50,17 @@ public class Component implements Serializable { private String name = null; private List<String> dependencies = new ArrayList<String>(); private ReadinessCheck readinessCheck = null; - private Artifact artifact = null; + private Artifact artifact = new Artifact(); private String launchCommand = null; - private Resource resource = null; + private Resource resource = new Resource(); private Long numberOfContainers = null; - private Boolean uniqueComponentSupport = null; - private Boolean runPrivilegedContainer = null; + private Boolean uniqueComponentSupport = false; + private Boolean runPrivilegedContainer = false; private PlacementPolicy placementPolicy = null; - private Configuration configuration = null; + private Configuration configuration = new Configuration(); private List<String> quicklinks = new ArrayList<String>(); + private List<Container> containers = + Collections.synchronizedList(new ArrayList<Container>()); /** * Name of the application component (mandatory). @@ -196,6 +199,29 @@ public class Component implements Serializable { this.numberOfContainers = numberOfContainers; } + @ApiModelProperty(example = "null", value = "Containers of a started component. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.") + @JsonProperty("containers") + public List<Container> getContainers() { + return containers; + } + + public void setContainers(List<Container> containers) { + this.containers = containers; + } + + public void addContainer(Container container) { + this.containers.add(container); + } + + public Container getContainer(String id) { + for (Container container : containers) { + if (container.getId().equals(id)) { + return container; + } + } + return null; + } + /** * Certain applications need to define multiple components using the same * artifact and resource profile, differing only in configurations. In such @@ -354,6 +380,8 @@ public class Component implements Serializable { sb.append(" resource: ").append(toIndentedString(resource)).append("\n"); sb.append(" numberOfContainers: ") .append(toIndentedString(numberOfContainers)).append("\n"); + sb.append(" containers: ").append(toIndentedString(containers)) + .append("\n"); sb.append(" uniqueComponentSupport: ") .append(toIndentedString(uniqueComponentSupport)).append("\n"); sb.append(" runPrivilegedContainer: ") http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java index bad68c1..cdc96b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java @@ -21,6 +21,7 @@ import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import java.io.Serializable; +import java.util.Map; import java.util.Objects; import javax.xml.bind.annotation.XmlElement; @@ -62,7 +63,7 @@ public class ConfigFile implements Serializable { private TypeEnum type = null; private String destFile = null; private String srcFile = null; - private Object props = null; + private Map<String, String> props = null; /** * Config file in the standard format like xml, properties, json, yaml, @@ -104,6 +105,8 @@ public class ConfigFile implements Serializable { } /** + * TODO this probably is not required for non-template configs. It is now used as symlink for localization for non-template configs - we could infer the name from destFile instead + * * Required for type template. This provides the source location of the * template which needs to be mounted as dest_file post property * substitutions. Typically the src_file would point to a source controlled @@ -131,21 +134,36 @@ public class ConfigFile implements Serializable { * src_file is mandatory and the src_file content is dumped to dest_file post * property substitutions. **/ - public ConfigFile props(Object props) { + public ConfigFile props(Map<String, String> props) { this.props = props; return this; } @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If the type is template then the attribute src_file is mandatory and the src_file content is dumped to dest_file post property substitutions.") @JsonProperty("props") - public Object getProps() { + public Map<String, String> getProps() { return props; } - public void setProps(Object props) { + public void setProps(Map<String, String> props) { this.props = props; } + public long getLong(String name, long defaultValue) { + if (name == null) { + return defaultValue; + } + String value = props.get(name.trim()); + return Long.parseLong(value); + } + + public boolean getBoolean(String name, boolean defaultValue) { + if (name == null) { + return defaultValue; + } + return Boolean.valueOf(props.get(name.trim())); + } + @Override public boolean equals(java.lang.Object o) { if (this == o) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java index c4f2ad4..c43bd64 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java @@ -29,6 +29,7 @@ import java.util.Objects; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.commons.lang.StringUtils; /** * Set of configuration properties that can be injected into the application @@ -104,6 +105,35 @@ public class Configuration implements Serializable { this.files = files; } + public long getPropertyLong(String name, long defaultValue) { + if (name == null) { + return defaultValue; + } + String value = properties.get(name.trim()); + if (StringUtils.isEmpty(value)) { + return defaultValue; + } + return Long.parseLong(value); + } + + public String getProperty(String name, String defaultValue) { + if (name == null) { + return defaultValue; + } + return properties.get(name.trim()); + } + + public void setProperty(String name, String value) { + properties.put(name, value); + } + + public String getProperty(String name) { + if (name == null) { + return null; + } + return properties.get(name.trim()); + } + @Override public boolean equals(java.lang.Object o) { if (this == o) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0536f18/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java index 190121d..c255369 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java @@ -39,7 +39,7 @@ public class Resource extends BaseResource implements Cloneable { private static final long serialVersionUID = -6431667797380250037L; private String profile = null; - private Integer cpus = null; + private Integer cpus = 1; private String memory = null; /** --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
