Repository: incubator-slider Updated Branches: refs/heads/develop 9f5dbb467 -> a29d825e6
SLIDER-555 AM needs to get log aggregation friendly log4j Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a29d825e Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a29d825e Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a29d825e Branch: refs/heads/develop Commit: a29d825e68b2f51dab80257ce3f4501cc2ce3e88 Parents: 9f5dbb4 Author: Gour Saha <[email protected]> Authored: Wed Nov 5 17:38:19 2014 -0800 Committer: Gour Saha <[email protected]> Committed: Wed Nov 5 17:38:19 2014 -0800 ---------------------------------------------------------------------- .../src/conf/log4j-server.properties | 69 ++++++++++++++++++++ slider-assembly/src/conf/log4j.properties | 2 +- .../org/apache/slider/client/SliderClient.java | 4 +- .../org/apache/slider/common/SliderKeys.java | 10 +++ .../apache/slider/common/tools/SliderUtils.java | 29 ++++++-- 5 files changed, 107 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a29d825e/slider-assembly/src/conf/log4j-server.properties ---------------------------------------------------------------------- diff --git a/slider-assembly/src/conf/log4j-server.properties b/slider-assembly/src/conf/log4j-server.properties new file mode 100644 index 0000000..90b5b1b --- /dev/null +++ b/slider-assembly/src/conf/log4j-server.properties @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is a log4j config for Slider appmaster + +log4j.rootLogger=INFO,stderr +log4j.threshhold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + +# log layout skips stack-trace creation operations by avoiding line numbers and method +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n + +# debug edition is much more expensive +#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n + +# configure stderr +# set the conversion pattern of stdout +# Print the date in ISO 8601 format +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.Target=System.err +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n + + +log4j.appender.subprocess=org.apache.log4j.ConsoleAppender +log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout +log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n +#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess + +# for debugging Slider +#log4j.logger.org.apache.slider=DEBUG + +# uncomment to debug service lifecycle issues +#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG +#log4j.logger.org.apache.hadoop.yarn.service=DEBUG + +# uncomment for YARN operations +#log4j.logger.org.apache.hadoop.yarn.client=DEBUG + +# uncomment this to debug security problems +#log4j.logger.org.apache.hadoop.security=DEBUG + +#crank back on some noise +log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR +log4j.logger.org.apache.hadoop.hdfs=WARN +log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR + + + +log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN +log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN +log4j.logger.org.apache.zookeeper=WARN +log4j.logger.org.apache.curator.framework.state=ERROR +log4j.logger.org.apache.curator.framework.imps=WARN http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a29d825e/slider-assembly/src/conf/log4j.properties ---------------------------------------------------------------------- diff --git a/slider-assembly/src/conf/log4j.properties b/slider-assembly/src/conf/log4j.properties index 823ffa3..fc5cc29 100644 --- a/slider-assembly/src/conf/log4j.properties +++ b/slider-assembly/src/conf/log4j.properties @@ -15,7 +15,7 @@ # limitations under the License. # -# log4j configuration used during build and unit tests +# log4j configuration used by Slider client (and during build and unit tests) log4j.rootLogger=INFO,stderr log4j.threshhold=ALL http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a29d825e/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index fcc6ef6..676d865 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -1188,9 +1188,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe confDir); } Path localConfDirPath = SliderUtils.createLocalPath(confDir); - log.debug("Slide configuration directory is {}; remote to be {}", - localConfDirPath, SliderKeys.SUBMITTED_CONF_DIR); remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR); + log.debug("Slider configuration directory is {}; remote to be {}", + localConfDirPath, remoteConfPath); SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null); } // the assumption here is that minimr cluster => this is a test run http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a29d825e/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index f11d200..1a94d83 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -133,6 +133,16 @@ public interface SliderKeys extends SliderXmlConfKeys { String SUBMITTED_CONF_DIR = "confdir"; /** + * Slider AM log4j file name + */ + String LOG4J_SERVER_PROP_FILENAME = "log4j-server.properties"; + + /** + * Standard log4j file name + */ + String LOG4J_PROP_FILENAME = "log4j.properties"; + + /** * name of the Slider client resource * loaded when the service is loaded. */ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a29d825e/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 82b17fd..0ff5ccb 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -415,7 +415,9 @@ public final class SliderUtils { new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath, permission); } - Path[] sourcePaths = new Path[srcFileCount]; + // Slider AM now has its own log4j-server.properties file + List<Path> sourcePaths = new ArrayList<Path>(); + Path serverLog4jPath = null; for (int i = 0; i < srcFileCount; i++) { FileStatus e = entries[i]; Path srcFile = e.getPath(); @@ -425,13 +427,32 @@ public final class SliderUtils { log.warn(msg); throw new IOException(msg); } + if (SliderKeys.LOG4J_SERVER_PROP_FILENAME.equals(srcFile.getName())) { + // log4j-server.properties will be copied as log4j.properties + serverLog4jPath = srcFile; + log.debug("copying src conf file {} as {}", srcFile, + SliderKeys.LOG4J_PROP_FILENAME); + continue; + } + if (SliderKeys.LOG4J_PROP_FILENAME.equals(srcFile.getName())) { + // ignore log4j.properties entirely for Slider AM + log.debug("ignoring file {}", srcFile); + continue; + } log.debug("copying src conf file {}", srcFile); - sourcePaths[i] = srcFile; + sourcePaths.add(srcFile); } - log.debug("Copying {} files from {} to dest {}", srcFileCount, + log.debug("Copying {} files from {} to dest {}", + sourcePaths.size() + (serverLog4jPath != null ? 1 : 0), srcDirPath, destDirPath); - FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf); + FileUtil.copy(srcFS, sourcePaths.toArray(new Path[0]), destFS, destDirPath, + false, true, conf); + // Now copy log4j-server.properties (if exists) to dest as log4j.properties + if (serverLog4jPath != null) { + FileUtil.copy(srcFS, serverLog4jPath, destFS, new Path(destDirPath, + SliderKeys.LOG4J_PROP_FILENAME), false, true, conf); + } return srcFileCount; }
