Updated Branches: refs/heads/master 522a691c9 -> 018d16984
CRUNCH-113: Move the log4j shenanigans out of MRPipeline and into crunch-test's DebugLogging and make the debug logging code more robust and configurable. Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/018d1698 Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/018d1698 Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/018d1698 Branch: refs/heads/master Commit: 018d16984f127a4a14ae3ab2d0af0ac839dfb439 Parents: 522a691 Author: Josh Wills <[email protected]> Authored: Tue Nov 13 13:23:41 2012 -0800 Committer: Josh Wills <[email protected]> Committed: Thu Jan 10 16:30:50 2013 -0800 ---------------------------------------------------------------------- crunch-test/pom.xml | 6 ++ .../java/org/apache/crunch/test/DebugLogging.java | 56 +++++++++++++++ crunch/pom.xml | 6 -- .../java/org/apache/crunch/impl/mr/MRPipeline.java | 15 ---- 4 files changed, 62 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/018d1698/crunch-test/pom.xml ---------------------------------------------------------------------- diff --git a/crunch-test/pom.xml b/crunch-test/pom.xml index 9185ef0..4953783 100644 --- a/crunch-test/pom.xml +++ b/crunch-test/pom.xml @@ -49,6 +49,12 @@ under the License. <scope>provided</scope> </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <scope>provided</scope> + </dependency> + <!-- Required by LocalJobRunner --> <dependency> <groupId>commons-io</groupId> http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/018d1698/crunch-test/src/main/java/org/apache/crunch/test/DebugLogging.java ---------------------------------------------------------------------- diff --git a/crunch-test/src/main/java/org/apache/crunch/test/DebugLogging.java b/crunch-test/src/main/java/org/apache/crunch/test/DebugLogging.java new file mode 100644 index 0000000..3bf2303 --- /dev/null +++ b/crunch-test/src/main/java/org/apache/crunch/test/DebugLogging.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.test; + +import org.apache.log4j.Appender; +import org.apache.log4j.ConsoleAppender; +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; + +/** + * Allows direct manipulation of the Hadoop log4j settings to aid in + * unit testing. Not recommended for production use. + */ +public final class DebugLogging { + + /** + * Enables logging Hadoop output to the console using the pattern + * '%-4r [%t] %-5p %c %x - %m%n' at the specified {@code Level}. + * + * @param level The log4j level + */ + public static void enable(Level level) { + enable(level, new ConsoleAppender(new PatternLayout("%-4r [%t] %-5p %c %x - %m%n"))); + } + + /** + * Enables logging to the given {@code Appender} at the specified {@code Level}. + * + * @param level The log4j level + * @param appender The log4j appender + */ + public static void enable(Level level, Appender appender) { + Logger hadoopLogger = LogManager.getLogger("org.apache.hadoop"); + hadoopLogger.setLevel(level); + hadoopLogger.addAppender(appender); + } + + private DebugLogging() { } +} http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/018d1698/crunch/pom.xml ---------------------------------------------------------------------- diff --git a/crunch/pom.xml b/crunch/pom.xml index f8993c1..3fcdacf 100644 --- a/crunch/pom.xml +++ b/crunch/pom.xml @@ -111,12 +111,6 @@ under the License. </dependency> <dependency> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - <scope>provided</scope> - </dependency> - - <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> <scope>provided</scope> http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/018d1698/crunch/src/main/java/org/apache/crunch/impl/mr/MRPipeline.java ---------------------------------------------------------------------- diff --git a/crunch/src/main/java/org/apache/crunch/impl/mr/MRPipeline.java b/crunch/src/main/java/org/apache/crunch/impl/mr/MRPipeline.java index 40c5863..4d8fa84 100644 --- a/crunch/src/main/java/org/apache/crunch/impl/mr/MRPipeline.java +++ b/crunch/src/main/java/org/apache/crunch/impl/mr/MRPipeline.java @@ -52,10 +52,6 @@ import org.apache.crunch.types.writable.WritableTypeFamily; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.log4j.Appender; -import org.apache.log4j.Level; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -344,17 +340,6 @@ public class MRPipeline implements Pipeline { public void enableDebug() { // Turn on Crunch runtime error catching. getConfiguration().setBoolean(RuntimeParameters.DEBUG, true); - - // Write Hadoop's WARN logs to the console. - Logger crunchInfoLogger = LogManager.getLogger("org.apache.crunch"); - Appender console = crunchInfoLogger.getAppender("A"); - if (console != null) { - Logger hadoopLogger = LogManager.getLogger("org.apache.hadoop"); - hadoopLogger.setLevel(Level.WARN); - hadoopLogger.addAppender(console); - } else { - LOG.warn("Could not find console appender named 'A' for writing Hadoop warning logs"); - } } @Override
