SLIDER-1138 pass memory and core reqs to unique components
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c7a173cd Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c7a173cd Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c7a173cd Branch: refs/heads/feature/SLIDER-1107_AM_config_generation Commit: c7a173cdcc15f675c1b4d471ff40eefb9afb1bae Parents: baf6532 Author: Billie Rinaldi <billie.rina...@gmail.com> Authored: Wed Jun 8 11:28:28 2016 -0700 Committer: Billie Rinaldi <billie.rina...@gmail.com> Committed: Wed Jun 8 11:28:28 2016 -0700 ---------------------------------------------------------------------- .../slider/server/appmaster/state/AppState.java | 26 +++++++++++--------- .../appstate/TestMockAppStateUniqueNames.groovy | 22 +++++++++++++++++ 2 files changed, 36 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7a173cd/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index ed87b89..b4465a8 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -802,6 +802,7 @@ public class AppState { continue; } if (hasUniqueNames(resources, name)) { + // THIS NAME IS A GROUP int desiredInstanceCount = getDesiredInstanceCount(resources, name); Integer groupCount = 0; if (groupCounts.containsKey(name)) { @@ -853,21 +854,21 @@ public class AppState { /** * Get the desired instance count of a role, rejecting negative values * @param resources resource map - * @param role role name + * @param roleGroup role group * @return the instance count * @throws BadConfigException if the count is negative */ private int getDesiredInstanceCount(ConfTreeOperations resources, - String role) throws BadConfigException { + String roleGroup) throws BadConfigException { int desiredInstanceCount = - resources.getComponentOptInt(role, COMPONENT_INSTANCES, 0); + resources.getComponentOptInt(roleGroup, COMPONENT_INSTANCES, 0); if (desiredInstanceCount < 0) { - log.error("Role {} has negative desired instances : {}", role, + log.error("Role {} has negative desired instances : {}", roleGroup, desiredInstanceCount); throw new BadConfigException( "Negative instance count (%) requested for component %s", - desiredInstanceCount, role); + desiredInstanceCount, roleGroup); } return desiredInstanceCount; } @@ -1353,12 +1354,12 @@ public class AppState { * @throws NumberFormatException if the role could not be parsed. */ private int getResourceRequirement(ConfTreeOperations resources, - String name, + String group, String option, int defVal, int maxVal) { - String val = resources.getComponentOpt(name, option, + String val = resources.getComponentOpt(group, option, Integer.toString(defVal)); Integer intVal; if (YARN_RESOURCE_MAX.equals(val)) { @@ -1380,14 +1381,15 @@ public class AppState { public Resource buildResourceRequirements(RoleStatus role, Resource capability) { // Set up resource requirements from role values String name = role.getName(); + String group = role.getGroup(); ConfTreeOperations resources = getResourcesSnapshot(); int cores = getResourceRequirement(resources, - name, + group, YARN_CORES, DEF_YARN_CORES, containerMaxCores); capability.setVirtualCores(cores); - int ram = getResourceRequirement(resources, name, + int ram = getResourceRequirement(resources, group, YARN_MEMORY, DEF_YARN_MEMORY, containerMaxMemory); @@ -1959,13 +1961,13 @@ public class AppState { /** * Get the node failure threshold for a specific role, falling back to * the global one if not - * @param roleName role name + * @param roleGroup role group * @return the threshold for failures */ - private int getNodeFailureThresholdForRole(String roleName) { + private int getNodeFailureThresholdForRole(String roleGroup) { ConfTreeOperations resources = instanceDefinition.getResourceOperations(); - return resources.getComponentOptInt(roleName, + return resources.getComponentOptInt(roleGroup, NODE_FAILURE_THRESHOLD, nodeFailureThreshold); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7a173cd/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy index 5256163..d3ba798 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy @@ -71,6 +71,8 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest def opts = [ (ResourceKeys.COMPONENT_INSTANCES): "1", (ResourceKeys.COMPONENT_PRIORITY) : "6", + (ResourceKeys.YARN_MEMORY) : "1024", + (ResourceKeys.YARN_CORES) : "2", (ResourceKeys.UNIQUE_NAMES) : "true", ] @@ -90,6 +92,8 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest def opts = [ (ResourceKeys.COMPONENT_INSTANCES): "2", (ResourceKeys.COMPONENT_PRIORITY): "7", + (ResourceKeys.YARN_MEMORY) : "384", + (ResourceKeys.YARN_CORES) : "4", (ResourceKeys.UNIQUE_NAMES) : "true", ] @@ -103,6 +107,12 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest assert 6 == appState.lookupRoleStatus("group11").priority assert 7 == appState.lookupRoleStatus("group21").priority assert 8 == appState.lookupRoleStatus("group22").priority + assert 1024 == appState.lookupRoleStatus("group11").resourceRequirements.memory + assert 384 == appState.lookupRoleStatus("group21").resourceRequirements.memory + assert 384 == appState.lookupRoleStatus("group22").resourceRequirements.memory + assert 2 == appState.lookupRoleStatus("group11").resourceRequirements.virtualCores + assert 4 == appState.lookupRoleStatus("group21").resourceRequirements.virtualCores + assert 4 == appState.lookupRoleStatus("group22").resourceRequirements.virtualCores } @Test @@ -111,6 +121,8 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest def opts = [ (ResourceKeys.COMPONENT_INSTANCES): "0", (ResourceKeys.COMPONENT_PRIORITY) : "6", + (ResourceKeys.YARN_MEMORY) : "384", + (ResourceKeys.YARN_CORES) : "4", (ResourceKeys.UNIQUE_NAMES) : "true", ] @@ -121,6 +133,8 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest appState.lookupRoleStatus(6) assert 0 == appState.lookupRoleStatus("group11").desired assert 6 == appState.lookupRoleStatus("group11").priority + assert 384 == appState.lookupRoleStatus("group11").resourceRequirements.memory + assert 4 == appState.lookupRoleStatus("group11").resourceRequirements.virtualCores } @Test @@ -129,6 +143,8 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest def opts = [ (ResourceKeys.COMPONENT_INSTANCES): "3", (ResourceKeys.COMPONENT_PRIORITY) : "6", + (ResourceKeys.YARN_MEMORY) : "640", + (ResourceKeys.YARN_CORES) : "8", (ResourceKeys.UNIQUE_NAMES) : "true", ] @@ -145,6 +161,12 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest assert 6 == appState.lookupRoleStatus("group11").priority assert 7 == appState.lookupRoleStatus("group12").priority assert 8 == appState.lookupRoleStatus("group13").priority + assert 640 == appState.lookupRoleStatus("group11").resourceRequirements.memory + assert 640 == appState.lookupRoleStatus("group12").resourceRequirements.memory + assert 640 == appState.lookupRoleStatus("group13").resourceRequirements.memory + assert 8 == appState.lookupRoleStatus("group11").resourceRequirements.virtualCores + assert 8 == appState.lookupRoleStatus("group12").resourceRequirements.virtualCores + assert 8 == appState.lookupRoleStatus("group13").resourceRequirements.virtualCores } }
