Repository: incubator-slider Updated Branches: refs/heads/develop 8850d062b -> 8267370dc
SLIDER-1201 Slider should make resource normalization configurable for app-owners Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8267370d Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8267370d Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8267370d Branch: refs/heads/develop Commit: 8267370dc2b843b29c77a9926eec94faa2803694 Parents: 8850d06 Author: Gour Saha <gourks...@apache.org> Authored: Thu Feb 2 13:57:49 2017 -0800 Committer: Gour Saha <gourks...@apache.org> Committed: Thu Feb 2 13:59:34 2017 -0800 ---------------------------------------------------------------------- .../org/apache/slider/api/ResourceKeys.java | 8 +++ .../apache/slider/common/tools/SliderUtils.java | 20 ++++++++ .../server/appmaster/SliderAppMaster.java | 21 ++++---- .../slider/server/appmaster/state/AppState.java | 13 +++-- ...tRoleHistoryOutstandingRequestTracker.groovy | 52 ++++++++++++++++++++ .../slider/common/tools/TestSliderUtils.java | 17 +++++++ 6 files changed, 117 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8267370d/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java index 92890be..d6d8789 100644 --- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java +++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java @@ -72,6 +72,14 @@ public interface ResourceKeys { String YARN_CORES = "yarn.vcores"; /** + * If normalization is set to false, then if the resource (memory and/or + * vcore) requested by a role is higher than YARN limits, then the resource + * request is not normalized. If this causes failures at the YARN level then + * applications are expecting that to happen. Default value is false. + */ + String YARN_RESOURCE_NORMALIZATION_ENABLED = "yarn.resource.normalization.enabled"; + + /** * Number of disks per instance to ask YARN for * {@value} */ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8267370d/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index e9f65ba..9638408 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -513,6 +513,26 @@ public final class SliderUtils { } /** + * Extract the first line of a multi-line string. This is typically used to + * prune the stack trace appended to the end of exception messages returned by + * YARN in AMRMClientAsync callbacks. + * + * @param msg + * message string (most likely multi-lines) + * @return the first line of a multi-line string or the original string if it + * is a null, empty or single-line + */ + public static String extractFirstLine(String msg) { + if (StringUtils.isNotBlank(msg)) { + int newlineIndex = msg.indexOf(System.lineSeparator()); + if (newlineIndex != -1) { + msg = msg.substring(0, newlineIndex); + } + } + return msg; + } + + /** * Create a configuration with Slider-specific tuning. * This is done rather than doing custom configs. * @return the config http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8267370d/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 8232225..1337ec5 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -27,6 +27,7 @@ import com.google.common.base.Preconditions; import com.google.protobuf.BlockingService; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.Path; @@ -2060,14 +2061,13 @@ public class SliderAppMaster extends AbstractSliderLaunchedService @Override //AMRMClientAsync public void onError(Throwable e) { - //callback says it's time to finish + // callback says it's time to finish LOG_YARN.error("AMRMClientAsync.onError() received {}", e, e); - signalAMComplete(new ActionStopSlider("stop", - EXIT_EXCEPTION_THROWN, + signalAMComplete(new ActionStopSlider("stop", EXIT_EXCEPTION_THROWN, FinalApplicationStatus.FAILED, - "AMRMClientAsync.onError() received " + e)); + SliderUtils.extractFirstLine(e.getLocalizedMessage()))); } - + /* =================================================================== */ /* RMOperationHandlerActions */ /* =================================================================== */ @@ -2356,13 +2356,13 @@ public class SliderAppMaster extends AbstractSliderLaunchedService /** * Handle any exception in a thread. If the exception provides an exit - * code, that is the one that will be used + * code, that is the one that will be used. * @param thread thread throwing the exception * @param exception exception */ public void onExceptionInThread(Thread thread, Throwable exception) { log.error("Exception in {}: {}", thread.getName(), exception, exception); - + // if there is a teardown in progress, ignore it if (amCompletionFlag.get()) { log.info("Ignoring exception: shutdown in progress"); @@ -2371,10 +2371,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService if (exception instanceof ExitCodeProvider) { exitCode = ((ExitCodeProvider) exception).getExitCode(); } - signalAMComplete(new ActionStopSlider("stop", - exitCode, - FinalApplicationStatus.FAILED, - exception.toString())); + signalAMComplete( + new ActionStopSlider("stop", exitCode, FinalApplicationStatus.FAILED, + SliderUtils.extractFirstLine(exception.getLocalizedMessage()))); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8267370d/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index c5fd38c..3db9388 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -1385,9 +1385,10 @@ public class AppState { } /** - * Build up the resource requirements for this role from the - * cluster specification, including substituing max allowed values - * if the specification asked for it. + * Build up the resource requirements for this role from the cluster + * specification, including substituting max allowed values if the + * specification asked for it (except when + * {@link ResourceKeys#YARN_RESOURCE_NORMALIZATION_ENABLED} is set to false). * @param role role * @param capability capability to set up. A new one may be created * during normalization @@ -1409,6 +1410,12 @@ public class AppState { containerMaxMemory); capability.setMemory(ram); log.debug("Component {} has RAM={}, vCores ={}", name, ram, cores); + boolean normalize = resources.getComponentOptBool(group, + YARN_RESOURCE_NORMALIZATION_ENABLED, true); + if (!normalize) { + log.info("Resource normalization: disabled"); + return Resources.createResource(ram, cores); + } Resource normalized = recordFactory.normalize(capability, minResource, maxResource); if (!Resources.equals(normalized, capability)) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8267370d/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy index 0969824..7be01ad 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy @@ -18,10 +18,14 @@ package org.apache.slider.server.appmaster.model.history +import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.api.records.Resource +import org.apache.hadoop.yarn.util.resource.Resources +import org.apache.slider.api.ResourceKeys import org.apache.slider.providers.PlacementPolicy import org.apache.slider.providers.ProviderRole import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.model.mock.MockAppState import org.apache.slider.server.appmaster.model.mock.MockPriority import org.apache.slider.server.appmaster.model.mock.MockResource import org.apache.slider.server.appmaster.operations.AbstractRMOperation @@ -36,6 +40,7 @@ import org.apache.slider.server.appmaster.state.OutstandingRequestTracker import org.apache.slider.server.appmaster.state.RoleStatus import org.junit.Test +@Slf4j class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { public static final String WORKERS_LABEL = "workers" @@ -301,6 +306,53 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { assert yarnRequest.nodes.size() == 2 } + @Test + public void testBuildResourceRequirements() throws Throwable { + // Store original values + def resources = appState.getResourcesSnapshot() + def origMem = resources.getComponentOpt(role0Status.group, + ResourceKeys.YARN_MEMORY, null) + def origVcores = resources.getComponentOpt(role0Status.group, + ResourceKeys.YARN_CORES, null) + + // Resource values to be used for this test + def testMem = 32768 + def testVcores = 2 + resources.setComponentOpt(role0Status.group, ResourceKeys.YARN_MEMORY, + Integer.toString(testMem)); + resources.setComponentOpt(role0Status.group, ResourceKeys.YARN_CORES, + Integer.toString(testVcores)); + + // Test normalization disabled + log.info("Test normalization: disabled") + resources.setComponentOpt(role0Status.group, + ResourceKeys.YARN_RESOURCE_NORMALIZATION_ENABLED, "false"); + def requestedRes = new MockResource(testMem, testVcores) + def expectedRes = new MockResource(testMem, testVcores) + log.info("Resource requested: " + requestedRes) + def resFinal = appState.buildResourceRequirements(role0Status, + new MockResource()) + log.info("Resource actual: " + resFinal) + assert Resources.equals(expectedRes, resFinal) + + // Test normalization enabled + log.info("Test normalization: enabled") + resources.setComponentOpt(role0Status.group, + ResourceKeys.YARN_RESOURCE_NORMALIZATION_ENABLED, "true"); + expectedRes = new MockResource(MockAppState.RM_MAX_RAM, testVcores) + log.info("Resource requested: " + requestedRes) + resFinal = appState.buildResourceRequirements(role0Status, + new MockResource()) + log.info("Resource actual: " + resFinal) + assert Resources.equals(expectedRes, resFinal) + + // revert resource configuration to original value + resources.setComponentOpt(role0Status.group, ResourceKeys.YARN_MEMORY, + origMem); + resources.setComponentOpt(role0Status.group, ResourceKeys.YARN_CORES, + origVcores); + } + /** * Create a new request (always against host1) * @param r role status http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8267370d/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java b/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java index 20e72c0..deca7a8 100644 --- a/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java +++ b/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java @@ -156,4 +156,21 @@ public class TestSliderUtils { SliderUtils.write(testWriteFile, "test".getBytes("UTF-8"), true); Assert.assertTrue(FileUtils.readFileToString(testWriteFile, "UTF-8").equals("test")); } + + @Test + public void testExtractFirstLine() { + String firstLine = "hello"; + String msg = firstLine + "\n2nd line\n3rd line"; + Assert.assertEquals("Should be first line only", firstLine, + SliderUtils.extractFirstLine(msg)); + msg = ""; + Assert.assertEquals("Should be empty", msg, + SliderUtils.extractFirstLine(msg)); + msg = " "; + Assert.assertEquals("Should contain spaces only", msg, + SliderUtils.extractFirstLine(msg)); + msg = null; + Assert.assertEquals("Should be null", msg, + SliderUtils.extractFirstLine(msg)); + } }
