Repository: incubator-slider Updated Branches: refs/heads/develop e94aa2696 -> 333b63572
SLIDER-810 YARN config changes to enable partial logs upload for long running services (fix include pattern and tests) Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/333b6357 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/333b6357 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/333b6357 Branch: refs/heads/develop Commit: 333b63572fb50576444a3d09287c4af37096faf8 Parents: e94aa26 Author: Gour Saha <[email protected]> Authored: Tue May 19 22:38:12 2015 -0700 Committer: Gour Saha <[email protected]> Committed: Tue May 19 22:38:12 2015 -0700 ---------------------------------------------------------------------- .../slider/core/launch/AbstractLauncher.java | 10 +-- .../core/launch/TestAppMasterLauncher.java | 80 +++++++++++++++----- 2 files changed, 66 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/333b6357/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java index 3e9b50b..93aff08 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java @@ -343,14 +343,12 @@ public abstract class AbstractLauncher extends Configured { // patterns are left empty by the app owner in resources file if (StringUtils.isEmpty(logIncludePattern) && StringUtils.isEmpty(logExcludePattern)) { - logIncludePattern = "*"; + logIncludePattern = ".*"; logExcludePattern = ""; - } - if (StringUtils.isEmpty(logIncludePattern) + } else if (StringUtils.isEmpty(logIncludePattern) && StringUtils.isNotEmpty(logExcludePattern)) { - logIncludePattern = "*"; - } - if (StringUtils.isNotEmpty(logIncludePattern) + logIncludePattern = ".*"; + } else if (StringUtils.isNotEmpty(logIncludePattern) && StringUtils.isEmpty(logExcludePattern)) { logExcludePattern = ""; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/333b6357/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java index a71cee0..60af770 100644 --- a/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java +++ b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java @@ -18,12 +18,14 @@ package org.apache.slider.core.launch; +import java.lang.reflect.Method; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Set; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.slider.api.ResourceKeys; import org.apache.slider.client.SliderYarnClientImpl; @@ -38,6 +40,10 @@ public class TestAppMasterLauncher { YarnClientApplication yarnClientApp; ApplicationSubmissionContext appSubmissionContext; Set<String> tags = Collections.emptySet(); + AppMasterLauncher appMasterLauncher = null; + boolean isOldApi = true; + Method rolledLogsIncludeMethod = null; + Method rolledLogsExcludeMethod = null; @Before public void initialize() throws Exception { @@ -49,6 +55,18 @@ public class TestAppMasterLauncher { .andReturn(appSubmissionContext).once(); EasyMock.expect(mockYarnClient.createApplication()) .andReturn(yarnClientApp).once(); + + try { + LogAggregationContext.class.getMethod("newInstance", String.class, + String.class, String.class, String.class); + isOldApi = false; + rolledLogsIncludeMethod = LogAggregationContext.class + .getMethod("getRolledLogsIncludePattern"); + rolledLogsExcludeMethod = LogAggregationContext.class + .getMethod("getRolledLogsExcludePattern"); + } catch (Exception e) { + isOldApi = true; + } } /** @@ -69,45 +87,71 @@ public class TestAppMasterLauncher { "command*.json| agent.log* | "); EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); - AppMasterLauncher appMasterLauncher = new AppMasterLauncher("cl1", - SliderKeys.APP_TYPE, null, null, mockYarnClient, false, null, options, - tags); + appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null, + null, mockYarnClient, false, null, options, tags); // Verify the include/exclude patterns String expectedInclude = "slider*.txt|agent.out"; - Assert.assertEquals(expectedInclude, - appMasterLauncher.logAggregationContext.getIncludePattern()); - String expectedExclude = "command*.json|agent.log*"; - Assert.assertEquals(expectedExclude, - appMasterLauncher.logAggregationContext.getExcludePattern()); + assertPatterns(expectedInclude, expectedExclude); EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); + } @Test public void testExtractLogAggregationContextEmptyIncludePattern() - throws Exception { + throws Exception { Map<String, String> options = new HashMap<String, String>(); options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, " "); options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, "command*.json| agent.log* | "); EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); - AppMasterLauncher appMasterLauncher = new AppMasterLauncher("cl1", - SliderKeys.APP_TYPE, null, null, mockYarnClient, false, null, options, - tags); + appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null, + null, mockYarnClient, false, null, options, tags); // Verify the include/exclude patterns - String expectedInclude = ""; - Assert.assertEquals(expectedInclude, - appMasterLauncher.logAggregationContext.getIncludePattern()); - + String expectedInclude = isOldApi ? "" : ".*"; String expectedExclude = "command*.json|agent.log*"; - Assert.assertEquals(expectedExclude, - appMasterLauncher.logAggregationContext.getExcludePattern()); + assertPatterns(expectedInclude, expectedExclude); EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); } + @Test + public void testExtractLogAggregationContextEmptyIncludeAndExcludePattern() + throws Exception { + Map<String, String> options = new HashMap<String, String>(); + options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, ""); + options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, " "); + + EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); + appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null, + null, mockYarnClient, false, null, options, tags); + + // Verify the include/exclude patterns + String expectedInclude = isOldApi ? "" : ".*"; + String expectedExclude = ""; + assertPatterns(expectedInclude, expectedExclude); + + EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); + } + + private void assertPatterns(String expectedIncludePattern, + String expectedExcludePattern) throws Exception { + if (isOldApi) { + Assert.assertEquals(expectedIncludePattern, + appMasterLauncher.logAggregationContext.getIncludePattern()); + Assert.assertEquals(expectedExcludePattern, + appMasterLauncher.logAggregationContext.getExcludePattern()); + } else { + Assert.assertEquals(expectedIncludePattern, + (String) rolledLogsIncludeMethod + .invoke(appMasterLauncher.logAggregationContext)); + Assert.assertEquals(expectedExcludePattern, + (String) rolledLogsExcludeMethod + .invoke(appMasterLauncher.logAggregationContext)); + } + } }
