Repository: hadoop Updated Branches: refs/heads/trunk 3c9d97b8f -> 42f3a7082
YARN-8877. [CSI] Extend service spec to allow setting resource attributes. Contributed by Weiwei Yang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/42f3a708 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/42f3a708 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/42f3a708 Branch: refs/heads/trunk Commit: 42f3a7082a90bc71f0e86dc1e50b0c77b05489cf Parents: 3c9d97b Author: Sunil G <sun...@apache.org> Authored: Mon Nov 12 21:09:30 2018 +0530 Committer: Sunil G <sun...@apache.org> Committed: Mon Nov 12 21:09:30 2018 +0530 ---------------------------------------------------------------------- .../yarn/api/records/ResourceInformation.java | 7 +++ .../api/records/ResourceInformation.java | 18 ++++++++ .../yarn/service/component/Component.java | 3 +- .../hadoop/yarn/service/TestServiceAM.java | 47 ++++++++++++++++++++ .../yarn/service/conf/ExampleAppJson.java | 1 + .../yarn/service/conf/TestAppJsonResolve.java | 18 ++++++++ .../yarn/service/conf/examples/external3.json | 26 +++++++++++ .../markdown/yarn-service/YarnServiceAPI.md | 2 +- 8 files changed, 120 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f3a708/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 057e94e..047c09a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -276,6 +276,13 @@ public class ResourceInformation implements Comparable<ResourceInformation> { } public static ResourceInformation newInstance(String name, String units, + long value, Map<String, String> attributes) { + return ResourceInformation + .newInstance(name, units, value, ResourceTypes.COUNTABLE, 0L, + Long.MAX_VALUE, null, attributes); + } + + public static ResourceInformation newInstance(String name, String units, ResourceTypes resourceType) { return ResourceInformation.newInstance(name, units, 0L, resourceType, 0L, Long.MAX_VALUE); http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f3a708/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java index 103fffb..e466ce7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ResourceInformation.java @@ -18,10 +18,12 @@ package org.apache.hadoop.yarn.service.api.records; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableMap; import com.google.gson.annotations.SerializedName; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import java.util.Map; import java.util.Objects; /** @@ -35,11 +37,25 @@ public class ResourceInformation { @SerializedName("unit") private String unit = null; + @SerializedName("attributes") + private Map<String, String> attributes = null; + public ResourceInformation value(Long value) { this.value = value; return this; } + @ApiModelProperty(value = "") + @JsonProperty("attributes") + public Map<String, String> getAttributes() { + return attributes == null ? ImmutableMap.of() : attributes; + } + + public ResourceInformation attributes(Map<String, String> attributes) { + this.attributes = attributes; + return this; + } + /** * Integer value of the resource. * @@ -98,6 +114,8 @@ public class ResourceInformation { sb.append("class ResourceInformation {\n"); sb.append(" value: ").append(toIndentedString(value)).append("\n"); sb.append(" unit: ").append(toIndentedString(unit)).append("\n"); + sb.append(" attributes: ").append(toIndentedString(attributes)) + .append("\n"); sb.append("}"); return sb.toString(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f3a708/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java index 526bde0..7833b06 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java @@ -701,7 +701,8 @@ public class Component implements EventHandler<ComponentEvent> { org.apache.hadoop.yarn.api.records.ResourceInformation.newInstance( entry.getKey(), specInfo.getUnit(), - specInfo.getValue()); + specInfo.getValue(), + specInfo.getAttributes()); resource.setResourceInformation(resourceName, ri); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f3a708/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java index 80f4910..51c27e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java @@ -456,4 +456,51 @@ public class TestServiceAM extends ServiceTestUtils{ Assert.fail("Fail to sync sysfs."); } } + + @Test + public void testScheduleWithResourceAttributes() throws Exception { + ApplicationId applicationId = ApplicationId.newInstance(123456, 1); + Service exampleApp = new Service(); + exampleApp.setId(applicationId.toString()); + exampleApp.setName("testScheduleWithResourceAttributes"); + exampleApp.setVersion("v1"); + + List<ResourceTypeInfo> resourceTypeInfos = new ArrayList<>( + ResourceUtils.getResourcesTypeInfo()); + // Add 3rd resource type. + resourceTypeInfos.add(ResourceTypeInfo + .newInstance("test-resource", "", ResourceTypes.COUNTABLE)); + // Reinitialize resource types + ResourceUtils.reinitializeResources(resourceTypeInfos); + + Component serviceCompoent = createComponent("compa", 1, "pwd"); + serviceCompoent.getResource().setResourceInformations( + ImmutableMap.of("test-resource", + new ResourceInformation() + .value(1234L) + .unit("Gi") + .attributes(ImmutableMap.of("k1", "v1", "k2", "v2")))); + exampleApp.addComponent(serviceCompoent); + + MockServiceAM am = new MockServiceAM(exampleApp); + am.init(conf); + am.start(); + + ServiceScheduler serviceScheduler = am.context.scheduler; + AMRMClientAsync<AMRMClient.ContainerRequest> amrmClientAsync = + serviceScheduler.getAmRMClient(); + + Collection<AMRMClient.ContainerRequest> rr = + amrmClientAsync.getMatchingRequests(0); + Assert.assertEquals(1, rr.size()); + + org.apache.hadoop.yarn.api.records.Resource capability = + rr.iterator().next().getCapability(); + Assert.assertEquals(1234L, capability.getResourceValue("test-resource")); + Assert.assertEquals("Gi", + capability.getResourceInformation("test-resource").getUnits()); + Assert.assertEquals(2, capability.getResourceInformation("test-resource") + .getAttributes().size()); + am.stop(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f3a708/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java index 5fdd2ab..754b589 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java @@ -38,6 +38,7 @@ public final class ExampleAppJson { public static final String EXTERNAL_JSON_0 = "external0.json"; public static final String EXTERNAL_JSON_1 = "external1.json"; public static final String EXTERNAL_JSON_2 = "external2.json"; + public static final String EXTERNAL_JSON_3 = "external3.json"; public static final String PACKAGE = "/org/apache/hadoop/yarn/service/conf/examples/"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f3a708/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java index 73f7fa1..25c502f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java @@ -20,7 +20,9 @@ package org.apache.hadoop.yarn.service.conf; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.service.ServiceTestUtils; +import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.Resource; +import org.apache.hadoop.yarn.service.api.records.ResourceInformation; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ConfigFile; import org.apache.hadoop.yarn.service.api.records.Configuration; @@ -214,4 +216,20 @@ public class TestAppJsonResolve extends Assert { other = orig.getComponent("other").getConfiguration(); assertEquals(0, other.getProperties().size()); } + + @Test + public void testSetResourceAttributes() throws IOException { + Service orig = ExampleAppJson.loadResource(EXTERNAL_JSON_3); + Component component = orig.getComponent("volume-service"); + Assert.assertNotNull(component); + Map<String, ResourceInformation> adResource = component + .getResource().getAdditional(); + Assert.assertNotNull(adResource); + Assert.assertEquals(1, adResource.size()); + Map.Entry<String, ResourceInformation> volume = adResource + .entrySet().iterator().next(); + Assert.assertEquals("yarn.io/csi-volume", volume.getKey()); + Assert.assertEquals(100L, volume.getValue().getValue().longValue()); + Assert.assertEquals(2, volume.getValue().getAttributes().size()); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f3a708/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external3.json ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external3.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external3.json new file mode 100644 index 0000000..74569bd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/external3.json @@ -0,0 +1,26 @@ +{ + "name": "external-3", + "version": "1.0.0", + "lifetime": "3600", + "components": [ + { + "name": "volume-service", + "launch_command": "sleep 3600", + "number_of_containers": 1, + "resource": { + "": 1, + "memory": "512", + "additional": { + "yarn.io/csi-volume": { + "value": 100, + "unit": "Gi", + "attributes" : { + "driver" : "hostpath", + "mountPath" : "/mnt/data" + } + } + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/42f3a708/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md index fe49158..4ee15e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md @@ -371,7 +371,7 @@ Resource determines the amount of resources (vcores, memory, network, etc.) usab |profile|Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.|false|string|| |cpus|Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).|false|integer (int32)|| |memory|Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.|false|string|| -|additional|A map of resource type name to resource type information. Including value (integer), and unit (string). This will be used to specify resource other than cpu and memory. Please refer to example below.|false|object|| +|additional|A map of resource type name to resource type information. Including value (integer), unit (string) and optional attributes (map). This will be used to specify resource other than cpu and memory. Please refer to example below.|false|object|| ### ResourceInformation --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
