Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-82-pass-3.1 e0fb52916 -> 5a61b4cd8
SLIDER-966 fix regressions shown up by tests, mostly in test setup, but one test, TestMockContainerResourceAllocations, showed that resource normalization stamped on vcore requirements Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/89fd701b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/89fd701b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/89fd701b Branch: refs/heads/feature/SLIDER-82-pass-3.1 Commit: 89fd701bf1ab50b759babb01b075307ed6080c0d Parents: e0fb529 Author: Steve Loughran <[email protected]> Authored: Mon Nov 9 13:58:03 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Mon Nov 9 13:58:03 2015 +0000 ---------------------------------------------------------------------- .../state/AbstractClusterServices.java | 24 +++++++++++- .../slider/server/appmaster/state/AppState.java | 15 +++++++- .../slider/client/TestClientBadArgs.groovy | 39 ++++++++++++-------- .../appmaster/model/mock/MockAppState.groovy | 5 ++- 4 files changed, 63 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/89fd701b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java index eba8c38..54f384b 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java @@ -18,6 +18,7 @@ package org.apache.slider.server.appmaster.state; +import com.google.common.base.Preconditions; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; @@ -25,6 +26,10 @@ import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; * Cluster services offered by the YARN infrastructure. */ public abstract class AbstractClusterServices { + + private final DefaultResourceCalculator + defaultResourceCalculator = new DefaultResourceCalculator(); + /** * Create a resource for requests * @return a resource which can be built up. @@ -33,7 +38,24 @@ public abstract class AbstractClusterServices { public abstract Resource newResource(int memory, int cores); + /** + * Normalise memory, CPU and other resources according to the YARN AM-supplied + * values and the resource calculator in use (currently hard-coded to the + * {@link DefaultResourceCalculator}. + * Those resources which aren't normalized (currently: CPU) are left + * as is. + * @param resource resource requirements of a role + * @param minR minimum values of this queue + * @param maxR max values of this queue + * @return a normalized value. + */ public Resource normalize(Resource resource, Resource minR, Resource maxR) { - return new DefaultResourceCalculator().normalize(resource, minR, maxR); + Preconditions.checkArgument(resource != null, "null resource"); + Preconditions.checkArgument(minR != null, "null minR"); + Preconditions.checkArgument(maxR != null, "null maxR"); + + Resource normalize = defaultResourceCalculator.normalize(resource, minR, + maxR, minR); + return newResource(normalize.getMemory(), resource.getVirtualCores()); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/89fd701b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 21f59a1..f74fe98 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -21,6 +21,7 @@ package org.apache.slider.server.appmaster.state; import com.codahale.metrics.Metric; import com.codahale.metrics.MetricRegistry; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.Container; @@ -34,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterDescriptionKeys; import org.apache.slider.api.ClusterDescriptionOperations; @@ -322,6 +324,8 @@ public class AppState { */ public AppState(AbstractClusterServices recordFactory, MetricsAndMonitoring metricsAndMonitoring) { + Preconditions.checkArgument(recordFactory != null, "null recordFactory"); + Preconditions.checkArgument(metricsAndMonitoring != null, "null metricsAndMonitoring"); this.recordFactory = recordFactory; this.metricsAndMonitoring = metricsAndMonitoring; @@ -1310,7 +1314,16 @@ public class AppState { DEF_YARN_MEMORY, containerMaxMemory); capability.setMemory(ram); - return recordFactory.normalize(capability,minResource, maxResource); + log.debug("Component {} has RAM={}, vCores ={}", name, ram, cores); + Resource normalized = recordFactory.normalize(capability, minResource, + maxResource); + if (!Resources.equals(normalized, capability)) { + // resource requirements normalized to something other than asked for. + // LOG @ WARN so users can see why this is happening. + log.warn("Resource requirements of {} normalized" + + " from {} to {}", name, capability, normalized); + } + return normalized; } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/89fd701b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy index 12736e3..1b074dc 100644 --- a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy @@ -21,6 +21,7 @@ package org.apache.slider.client import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.common.params.ActionRegistryArgs import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions @@ -44,7 +45,7 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { @Test public void testNoAction() throws Throwable { launchExpectingException(SliderClient, - new Configuration(), + createTestConfig(), "Usage: slider COMMAND", []) @@ -53,7 +54,7 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { @Test public void testUnknownAction() throws Throwable { launchExpectingException(SliderClient, - new Configuration(), + createTestConfig(), "not-a-known-action", ["not-a-known-action"]) } @@ -61,7 +62,7 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { @Test public void testActionWithoutOptions() throws Throwable { launchExpectingException(SliderClient, - new Configuration(), + createTestConfig(), "Usage: slider build <application>", [SliderActions.ACTION_BUILD]) } @@ -69,7 +70,7 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { @Test public void testActionWithoutEnoughArgs() throws Throwable { launchExpectingException(SliderClient, - new Configuration(), + createTestConfig(), ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS, [SliderActions.ACTION_THAW]) } @@ -77,7 +78,7 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { @Test public void testActionWithTooManyArgs() throws Throwable { launchExpectingException(SliderClient, - new Configuration(), + createTestConfig(), ErrorStrings.ERROR_TOO_MANY_ARGUMENTS, [SliderActions.ACTION_HELP, "hello, world"]) @@ -86,7 +87,7 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { @Test public void testBadImageArg() throws Throwable { launchExpectingException(SliderClient, - new Configuration(), + createTestConfig(), "Unknown option: --image", [SliderActions.ACTION_HELP, Arguments.ARG_IMAGE]) @@ -95,7 +96,7 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { @Test public void testRegistryUsage() throws Throwable { def exception = launchExpectingException(SliderClient, - new Configuration(), + createTestConfig(), "org.apache.slider.core.exceptions.UsageException: Argument --name missing", [SliderActions.ACTION_REGISTRY]) assert exception instanceof UsageException @@ -105,7 +106,7 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { @Test public void testRegistryExportBadUsage1() throws Throwable { def exception = launchExpectingException(SliderClient, - new Configuration(), + createTestConfig(), "Expected a value after parameter --getexp", [SliderActions.ACTION_REGISTRY, Arguments.ARG_NAME, @@ -118,7 +119,7 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { @Test public void testRegistryExportBadUsage2() throws Throwable { def exception = launchExpectingException(SliderClient, - new Configuration(), + createTestConfig(), "Expected a value after parameter --getexp", [SliderActions.ACTION_REGISTRY, Arguments.ARG_NAME, @@ -132,7 +133,7 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { @Test public void testRegistryExportBadUsage3() throws Throwable { def exception = launchExpectingException(SliderClient, - new Configuration(), + createTestConfig(), "Usage: registry", [SliderActions.ACTION_REGISTRY, Arguments.ARG_NAME, @@ -147,7 +148,7 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { @Test public void testUpgradeUsage() throws Throwable { def exception = launchExpectingException(SliderClient, - new Configuration(), + createTestConfig(), "org.apache.slider.core.exceptions.BadCommandArgumentsException: Not enough arguments for action: upgrade Expected minimum 1 but got 0", [SliderActions.ACTION_UPGRADE]) assert exception instanceof BadCommandArgumentsException @@ -158,7 +159,7 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { public void testUpgradeWithTemplateOptionOnly() throws Throwable { String appName = "test_hbase" def exception = launchExpectingException(SliderClient, - new Configuration(), + createTestConfig(), "BadCommandArgumentsException: Option --resources must be specified with option --template", [SliderActions.ACTION_UPGRADE, appName, @@ -169,11 +170,17 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { log.info(exception.toString()) } + public Configuration createTestConfig() { + def configuration = new Configuration() + configuration.set(YarnConfiguration.RM_ADDRESS, "127.0.0.1:8032") + return configuration + } + @Test public void testUpgradeWithResourcesOptionOnly() throws Throwable { String appName = "test_hbase" def exception = launchExpectingException(SliderClient, - new Configuration(), + createTestConfig(), "BadCommandArgumentsException: Option --template must be specified with option --resources", [SliderActions.ACTION_UPGRADE, appName, @@ -188,7 +195,7 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { public void testUpgradeWithTemplateResourcesAndContainersOption() throws Throwable { String appName = "test_hbase" def exception = launchExpectingException(SliderClient, - new Configuration(), + createTestConfig(), "BadCommandArgumentsException: Option --containers cannot be " + "specified with --template or --resources", [SliderActions.ACTION_UPGRADE, @@ -208,7 +215,7 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { public void testUpgradeWithTemplateResourcesAndComponentsOption() throws Throwable { String appName = "test_hbase" def exception = launchExpectingException(SliderClient, - new Configuration(), + createTestConfig(), "BadCommandArgumentsException: Option --components cannot be " + "specified with --template or --resources", [SliderActions.ACTION_UPGRADE, @@ -228,7 +235,7 @@ class TestClientBadArgs extends ServiceLauncherBaseTest { public void testCreateAppWithAddonPkgBadArg1() throws Throwable { //add on package without specifying add on package name def exception = launchExpectingException(SliderClient, - new Configuration(), + createTestConfig(), "Expected 2 values after --addon", [SliderActions.ACTION_CREATE, "cl1", http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/89fd701b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy index 5565e6b..c041ce5 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy @@ -29,8 +29,10 @@ import org.apache.slider.server.appmaster.state.AppStateBindingInfo class MockAppState extends AppState { public static final int RM_MAX_RAM = 4096 public static final int RM_MAX_CORES = 64 + public MockAppState(AbstractClusterServices recordFactory) { super(recordFactory, new MetricsAndMonitoring()); + setContainerLimits(1, RM_MAX_RAM, 1, RM_MAX_CORES) } long time = 0; @@ -39,8 +41,7 @@ class MockAppState extends AppState { * Instance with a mock record factory */ public MockAppState() { - super(new MockClusterServices(), new MetricsAndMonitoring()); - setContainerLimits(1, RM_MAX_RAM, 1, RM_MAX_CORES) + this(new MockClusterServices()); } MockAppState(AppStateBindingInfo bindingInfo) {
