SLIDER-116. Log handling for long-lived applications (Gour Saha via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a34837ce Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a34837ce Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a34837ce Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option Commit: a34837ce1d0431c960f956251eb57e23054253d1 Parents: 314c3cc Author: Sumit Mohanty <[email protected]> Authored: Wed Sep 24 13:50:52 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Wed Sep 24 13:51:00 2014 -0700 ---------------------------------------------------------------------- app-packages/accumulo/resources.json | 3 + .../accumulo/src/test/resources/resources.json | 3 + app-packages/hbase-win/resources.json | 3 + .../hbase/src/test/resources/resources.json | 3 + app-packages/storm/resources.json | 3 + .../org/apache/slider/api/ResourceKeys.java | 3 + .../org/apache/slider/client/SliderClient.java | 2 + .../slider/core/launch/AbstractLauncher.java | 51 +++++++++ .../slider/core/launch/AppMasterLauncher.java | 4 +- .../core/launch/TestAppMasterLauncher.java | 112 +++++++++++++++++++ 10 files changed, 186 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/accumulo/resources.json ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/resources.json b/app-packages/accumulo/resources.json index f876901..f9cae60 100644 --- a/app-packages/accumulo/resources.json +++ b/app-packages/accumulo/resources.json @@ -3,6 +3,9 @@ "metadata": { }, "global": { + "yarn.log.include.patterns": "", + "yarn.log.exclude.patterns": "", + "yarn.log.interval": "0" }, "components": { "ACCUMULO_MASTER": { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/accumulo/src/test/resources/resources.json ---------------------------------------------------------------------- diff --git a/app-packages/accumulo/src/test/resources/resources.json b/app-packages/accumulo/src/test/resources/resources.json index 2711ba3..cfd9b3f 100644 --- a/app-packages/accumulo/src/test/resources/resources.json +++ b/app-packages/accumulo/src/test/resources/resources.json @@ -3,6 +3,9 @@ "metadata": { }, "global": { + "yarn.log.include.patterns": "", + "yarn.log.exclude.patterns": "", + "yarn.log.interval": "0" }, "components": { "ACCUMULO_MASTER": { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/hbase-win/resources.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/resources.json b/app-packages/hbase-win/resources.json index e0ff26f..eb38726 100644 --- a/app-packages/hbase-win/resources.json +++ b/app-packages/hbase-win/resources.json @@ -3,6 +3,9 @@ "metadata": { }, "global": { + "yarn.log.include.patterns": "", + "yarn.log.exclude.patterns": "", + "yarn.log.interval": "0" }, "components": { "HBASE_MASTER": { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/hbase/src/test/resources/resources.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase/src/test/resources/resources.json b/app-packages/hbase/src/test/resources/resources.json index e0ff26f..eb38726 100644 --- a/app-packages/hbase/src/test/resources/resources.json +++ b/app-packages/hbase/src/test/resources/resources.json @@ -3,6 +3,9 @@ "metadata": { }, "global": { + "yarn.log.include.patterns": "", + "yarn.log.exclude.patterns": "", + "yarn.log.interval": "0" }, "components": { "HBASE_MASTER": { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/storm/resources.json ---------------------------------------------------------------------- diff --git a/app-packages/storm/resources.json b/app-packages/storm/resources.json index 2805a3b..acecd91 100644 --- a/app-packages/storm/resources.json +++ b/app-packages/storm/resources.json @@ -3,6 +3,9 @@ "metadata" : { }, "global" : { + "yarn.log.include.patterns": "", + "yarn.log.exclude.patterns": "", + "yarn.log.interval": "0" }, "components": { "slider-appmaster": { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java index b542f1a..56961c9 100644 --- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java +++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java @@ -118,4 +118,7 @@ public interface ResourceKeys { */ int DEFAULT_CONTAINER_FAILURE_THRESHOLD = 5; + String YARN_LOG_INCLUDE_PATTERNS = "yarn.log.include.patterns"; + String YARN_LOG_EXCLUDE_PATTERNS = "yarn.log.exclude.patterns"; + String YARN_LOG_INTERVAL = "yarn.log.interval"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 129f851..afbb4a8 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -1032,6 +1032,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } MapOperations sliderAMResourceComponent = resourceOperations.getOrAddComponent(SliderKeys.COMPONENT_AM); + MapOperations resourceGlobalOptions = resourceOperations.getGlobalOptions(); // add the tags if available Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem, @@ -1043,6 +1044,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe yarnClient, clusterSecure, sliderAMResourceComponent, + resourceGlobalOptions, applicationTags); ApplicationId appId = amLauncher.getApplicationId(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java index 366fc8b..d3bb34c 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java @@ -19,6 +19,8 @@ package org.apache.slider.core.launch; import com.google.common.base.Preconditions; + +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; @@ -28,6 +30,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.util.Records; import org.apache.slider.api.ResourceKeys; @@ -70,6 +73,7 @@ public abstract class AbstractLauncher extends Configured { new HashMap<String, ByteBuffer>(); // security protected final Credentials credentials = new Credentials(); + protected LogAggregationContext logAggregationContext; protected AbstractLauncher(Configuration conf, @@ -258,6 +262,53 @@ public abstract class AbstractLauncher extends Configured { } } + public void extractLogAggregationContext(Map<String, String> map) { + if (map != null) { + String logPatternSepStr = "\\|"; + String logPatternJoinStr = "|"; + MapOperations options = new MapOperations("", map); + + List<String> logIncludePatterns = new ArrayList<String>(); + String includePatternExpression = options.getOption( + ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, "").trim(); + if (!includePatternExpression.isEmpty()) { + String[] includePatterns = includePatternExpression + .split(logPatternSepStr); + for (String includePattern : includePatterns) { + String trimmedIncludePattern = includePattern.trim(); + if (!trimmedIncludePattern.isEmpty()) { + logIncludePatterns.add(trimmedIncludePattern); + } + } + } + String logIncludePattern = StringUtils.join(logIncludePatterns, + logPatternJoinStr); + log.info("Log include patterns: {}", logIncludePattern); + + List<String> logExcludePatterns = new ArrayList<String>(); + String excludePatternExpression = options.getOption( + ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, "").trim(); + if (!excludePatternExpression.isEmpty()) { + String[] excludePatterns = excludePatternExpression + .split(logPatternSepStr); + for (String excludePattern : excludePatterns) { + String trimmedExcludePattern = excludePattern.trim(); + if (!trimmedExcludePattern.isEmpty()) { + logExcludePatterns.add(trimmedExcludePattern); + } + } + } + String logExcludePattern = StringUtils.join(logExcludePatterns, + logPatternJoinStr); + log.info("Log exclude patterns: {}", logExcludePattern); + + long logInterval = options + .getOptionInt(ResourceKeys.YARN_LOG_INTERVAL, 0); + log.info("Log interval: {}", logInterval); + logAggregationContext = LogAggregationContext.newInstance( + logIncludePattern, logExcludePattern, logInterval); + } + } /** * Utility method to set up the classpath http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java index c9ac20c..f1eeabd 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java @@ -77,6 +77,7 @@ public class AppMasterLauncher extends AbstractLauncher { SliderYarnClientImpl yarnClient, boolean secureCluster, Map<String, String> options, + Map<String, String> resourceGlobalOptions, Set<String> applicationTags ) throws IOException, YarnException { super(conf, fs); @@ -96,7 +97,7 @@ public class AppMasterLauncher extends AbstractLauncher { submissionContext.setApplicationTags(applicationTags); } extractResourceRequirements(resource, options); - + extractLogAggregationContext(resourceGlobalOptions); } public void setMaxAppAttempts(int maxAppAttempts) { @@ -170,6 +171,7 @@ public class AppMasterLauncher extends AbstractLauncher { //container requirements submissionContext.setResource(resource); + submissionContext.setLogAggregationContext(logAggregationContext); if (keepContainersOverRestarts) { log.debug("Requesting cluster stays running over AM failure"); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java new file mode 100644 index 0000000..ca1e9ec --- /dev/null +++ b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.launch; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; +import org.apache.slider.api.ResourceKeys; +import org.apache.slider.client.SliderYarnClientImpl; +import org.apache.slider.common.SliderKeys; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestAppMasterLauncher { + SliderYarnClientImpl mockYarnClient; + YarnClientApplication yarnClientApp; + ApplicationSubmissionContext appSubmissionContext; + Set<String> tags = Collections.emptySet(); + + @Before + public void initialize() throws Exception { + mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class); + yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class); + appSubmissionContext = EasyMock + .createNiceMock(ApplicationSubmissionContext.class); + EasyMock.expect(yarnClientApp.getApplicationSubmissionContext()) + .andReturn(appSubmissionContext).once(); + EasyMock.expect(mockYarnClient.createApplication()) + .andReturn(yarnClientApp).once(); + } + + @Test + public void testExtractLogAggregationContext() throws Exception { + Map<String, String> options = new HashMap<String, String>(); + options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, + " | slider*.txt |agent.out| |"); + options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, + "command*.json| agent.log* | "); + options.put(ResourceKeys.YARN_LOG_INTERVAL, "30"); + + EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); + AppMasterLauncher appMasterLauncher = new AppMasterLauncher("cl1", + SliderKeys.APP_TYPE, null, null, mockYarnClient, false, null, options, + tags); + + // Verify the include/exclude patterns + String expectedInclude = "slider*.txt|agent.out"; + Assert.assertEquals(expectedInclude, + appMasterLauncher.logAggregationContext.getIncludePattern()); + + String expectedExclude = "command*.json|agent.log*"; + Assert.assertEquals(expectedExclude, + appMasterLauncher.logAggregationContext.getExcludePattern()); + + Assert.assertEquals(30, + appMasterLauncher.logAggregationContext.getRollingIntervalSeconds()); + + EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); + } + + @Test + public void testExtractLogAggregationContextEmptyIncludePattern() + throws Exception { + Map<String, String> options = new HashMap<String, String>(); + options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, " "); + options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, + "command*.json| agent.log* | "); + options.put(ResourceKeys.YARN_LOG_INTERVAL, "600"); + + EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); + AppMasterLauncher appMasterLauncher = new AppMasterLauncher("cl1", + SliderKeys.APP_TYPE, null, null, mockYarnClient, false, null, options, + tags); + + // Verify the include/exclude patterns + String expectedInclude = ""; + Assert.assertEquals(expectedInclude, + appMasterLauncher.logAggregationContext.getIncludePattern()); + + String expectedExclude = "command*.json|agent.log*"; + Assert.assertEquals(expectedExclude, + appMasterLauncher.logAggregationContext.getExcludePattern()); + + Assert.assertEquals(600, + appMasterLauncher.logAggregationContext.getRollingIntervalSeconds()); + + EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); + } + +}
