This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 58e559b YARN-9354. Resources should be created with
ResourceTypesTestHelper instead of TestUtils. Contributed by Andras Gyori
58e559b is described below
commit 58e559b5ac1d54f6e0263e72d486a753e1e1fd9a
Author: Szilard Nemeth <snem...@apache.org>
AuthorDate: Wed Apr 15 08:16:15 2020 +0200
YARN-9354. Resources should be created with ResourceTypesTestHelper instead
of TestUtils. Contributed by Andras Gyori
---
.../resourcetypes/ResourceTypesTestHelper.java | 24 ++++++++++++++--------
.../TestApplicationMasterService.java | 22 ++++++++++----------
...estCapacitySchedulerWithMultiResourceTypes.java | 19 +++++++++--------
.../scheduler/capacity/TestUtils.java | 19 -----------------
4 files changed, 36 insertions(+), 48 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
index 3c3c2cc..198b395 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/resourcetypes/ResourceTypesTestHelper.java
@@ -57,14 +57,15 @@ public final class ResourceTypesTestHelper {
Resource resource = RECORD_FACTORY.newRecordInstance(Resource.class);
resource.setMemorySize(memory);
resource.setVirtualCores(vCores);
-
- for (Map.Entry<String, String> customResource :
- customResources.entrySet()) {
- String resourceName = customResource.getKey();
- ResourceInformation resourceInformation =
- createResourceInformation(resourceName,
- customResource.getValue());
- resource.setResourceInformation(resourceName, resourceInformation);
+ if (customResources != null) {
+ for (Map.Entry<String, String> customResource :
+ customResources.entrySet()) {
+ String resourceName = customResource.getKey();
+ ResourceInformation resourceInformation =
+ createResourceInformation(resourceName,
+ customResource.getValue());
+ resource.setResourceInformation(resourceName, resourceInformation);
+ }
}
return resource;
}
@@ -107,7 +108,12 @@ public final class ResourceTypesTestHelper {
public static Map<String, String> extractCustomResourcesAsStrings(
Resource res) {
Map<String, Long> resValues = extractCustomResources(res);
- return resValues.entrySet().stream()
+ return convertCustomResources(resValues);
+ }
+
+ public static Map<String, String> convertCustomResources(
+ Map<String, ? extends Number> customResources) {
+ return customResources.entrySet().stream()
.collect(Collectors.toMap(
Map.Entry::getKey, e -> String.valueOf(e.getValue())));
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index bfe908d..45c5ac8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -82,7 +82,6 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
-import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
@@ -746,8 +745,8 @@ public class TestApplicationMasterService {
MockRM rm = new MockRM(yarnConf);
rm.start();
- MockNM nm1 = rm.registerNode("199.99.99.1:1234", TestUtils
- .createResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ MockNM nm1 = rm.registerNode("199.99.99.1:1234", ResourceTypesTestHelper
+ .newResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, null));
RMApp app1 = rm.submitApp(GB, "app", "user", null, "default");
@@ -894,10 +893,10 @@ public class TestApplicationMasterService {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
LeafQueue leafQueue = (LeafQueue) cs.getQueue("default");
- MockNM nm1 = rm.registerNode("199.99.99.1:1234", TestUtils
- .createResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ MockNM nm1 = rm.registerNode("199.99.99.1:1234", ResourceTypesTestHelper
+ .newResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
- ImmutableMap.of("res_1", 4)));
+ ImmutableMap.of("res_1", "4")));
RMApp app1 = rm.submitApp(GB, "app", "user", null, "default");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
@@ -909,8 +908,8 @@ public class TestApplicationMasterService {
boolean exception = false;
try {
am1.allocate(Collections.singletonList(ResourceRequest.newBuilder()
- .capability(TestUtils.createResource(9 * GB, 1,
- ImmutableMap.of("res_1", 1)))
+ .capability(ResourceTypesTestHelper.newResource(9 * GB, 1,
+ ImmutableMap.of("res_1", "1")))
.numContainers(1)
.resourceName("*")
.build()), null);
@@ -924,7 +923,8 @@ public class TestApplicationMasterService {
// Now request resource, vcores > allowed
am1.allocate(Collections.singletonList(ResourceRequest.newBuilder()
.capability(
- TestUtils.createResource(8 * GB, 18, ImmutableMap.of("res_1",
1)))
+ ResourceTypesTestHelper.newResource(8 * GB, 18,
+ ImmutableMap.of("res_1", "1")))
.numContainers(1)
.resourceName("*")
.build()), null);
@@ -937,8 +937,8 @@ public class TestApplicationMasterService {
try {
// Now request resource, res_1 > allowed
am1.allocate(Collections.singletonList(ResourceRequest.newBuilder()
- .capability(TestUtils.createResource(8 * GB, 1,
- ImmutableMap.of("res_1", 100)))
+ .capability(ResourceTypesTestHelper.newResource(8 * GB, 1,
+ ImmutableMap.of("res_1", "100")))
.numContainers(1)
.resourceName("*")
.build()), null);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
index fa2b9935..de3673e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
@@ -294,16 +295,16 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
MockRM rm = new MockRM(conf);
rm.start();
- Map<String, Integer> nameToValues = new HashMap<>();
- nameToValues.put(ResourceInformation.GPU_URI, 4);
+ Map<String, String> nameToValues = new HashMap<>();
+ nameToValues.put(ResourceInformation.GPU_URI, "4");
// Register NM1 with 10GB memory, 4 CPU and 4 GPU
MockNM nm1 = rm.registerNode("127.0.0.1:1234",
- TestUtils.createResource(10 * GB, 4, nameToValues));
+ ResourceTypesTestHelper.newResource(10 * GB, 4, nameToValues));
nameToValues.clear();
// Register NM2 with 10GB memory, 4 CPU and 0 GPU
rm.registerNode("127.0.0.1:1235",
- TestUtils.createResource(10 * GB, 4, nameToValues));
+ ResourceTypesTestHelper.newResource(10 * GB, 4, nameToValues));
RMApp app1 = rm.submitApp(1024, "app-1", "user1", null, "a");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
@@ -320,9 +321,9 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
Assert.assertEquals(4, report_nm1.getAvailableResource()
.getResourceInformation(ResourceInformation.GPU_URI).getValue());
- nameToValues.put(ResourceInformation.GPU_URI, 4);
+ nameToValues.put(ResourceInformation.GPU_URI, "4");
Resource containerGpuResource =
- TestUtils.createResource(1 * GB, 1, nameToValues);
+ ResourceTypesTestHelper.newResource(1 * GB, 1, nameToValues);
// Allocate one container which takes all 4 GPU
am1.allocate(
@@ -345,7 +346,7 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
nameToValues.clear();
Resource containerResource =
- TestUtils.createResource(1 * GB, 1, nameToValues);
+ ResourceTypesTestHelper.newResource(1 * GB, 1, nameToValues);
// Allocate one more container which doesnt need GPU
am1.allocate(
Collections.singletonList(ResourceRequest.newInstance(
@@ -469,8 +470,8 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
for (int i = 0; i < 8; i++) {
fiCaApp1.updateResourceRequests(Collections.singletonList(
ResourceRequest.newBuilder()
- .capability(TestUtils.createResource(1 * GB, 1,
- ImmutableMap.of("res_1", 10)))
+ .capability(ResourceTypesTestHelper.newResource(1 * GB, 1,
+ ImmutableMap.of("res_1", "10")))
.numContainers(1)
.resourceName("*")
.build()));
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
index c00b92d..418f859 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
@@ -466,23 +466,4 @@ public class TestUtils {
cs.submitResourceCommitRequest(clusterResource,
csAssignment);
}
-
- /**
- * An easy way to create resources other than memory and vcores for tests.
- * @param memory memory
- * @param vcores vcores
- * @param nameToValues resource types other than memory and vcores.
- * @return created resource
- */
- public static Resource createResource(long memory, int vcores,
- Map<String, Integer> nameToValues) {
- Resource res = Resource.newInstance(memory, vcores);
- if (nameToValues != null) {
- for (Map.Entry<String, Integer> entry : nameToValues.entrySet()) {
- res.setResourceInformation(entry.getKey(), ResourceInformation
- .newInstance(entry.getKey(), "", entry.getValue()));
- }
- }
- return res;
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
