Repository: oozie Updated Branches: refs/heads/master 1600045ff -> 55171fc09
OOZIE-2430 Add root logger for hive,sqoop action Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/55171fc0 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/55171fc0 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/55171fc0 Branch: refs/heads/master Commit: 55171fc0949c76c6c9bb8ab0be6f4f7191831207 Parents: 1600045 Author: Purshotam Shah <[email protected]> Authored: Thu Jan 21 08:58:30 2016 -0800 Committer: Purshotam Shah <[email protected]> Committed: Thu Jan 21 08:58:30 2016 -0800 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 41 +++++++++++++++++--- core/src/main/resources/oozie-default.xml | 8 ++++ .../action/hadoop/TestJavaActionExecutor.java | 26 +++++++++++++ release-log.txt | 1 + .../apache/oozie/action/hadoop/HiveMain.java | 6 ++- .../oozie/action/hadoop/LauncherMapper.java | 1 + .../org/apache/oozie/action/hadoop/PigMain.java | 3 +- .../oozie/action/hadoop/PigMainWithOldAPI.java | 2 + .../apache/oozie/action/hadoop/SqoopMain.java | 2 + 9 files changed, 83 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index c7cbdae..4fc0c52 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -33,9 +33,9 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import java.util.Set; -import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -45,12 +45,15 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.AccessControlException; +import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.DiskChecker; import org.apache.oozie.WorkflowActionBean; import org.apache.oozie.WorkflowJobBean; @@ -67,6 +70,7 @@ import org.apache.oozie.service.Services; import org.apache.oozie.service.ShareLibService; import org.apache.oozie.service.URIHandlerService; import org.apache.oozie.service.WorkflowAppService; +import org.apache.oozie.util.ELEvaluationException; import org.apache.oozie.util.ELEvaluator; import org.apache.oozie.util.JobUtils; import org.apache.oozie.util.LogUtils; @@ -74,13 +78,9 @@ import org.apache.oozie.util.PropertiesUtils; import org.apache.oozie.util.XConfiguration; import org.apache.oozie.util.XLog; import org.apache.oozie.util.XmlUtils; -import org.apache.oozie.util.ELEvaluationException; import org.jdom.Element; import org.jdom.JDOMException; import org.jdom.Namespace; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.oozie.hadoop.utils.HadoopShims; public class JavaActionExecutor extends ActionExecutor { @@ -521,6 +521,7 @@ public class JavaActionExecutor extends ActionExecutor { // set cancel.delegation.token in actionConf that child job doesn't cancel delegation token actionConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false); updateConfForJavaTmpDir(actionConf); + setRootLoggerLevel(actionConf); return actionConf; } catch (IOException ex) { @@ -534,6 +535,34 @@ public class JavaActionExecutor extends ActionExecutor { } } + /** + * Set root log level property in actionConf + * @param actionConf + */ + void setRootLoggerLevel(Configuration actionConf) { + String oozieActionTypeRootLogger = "oozie.action." + getType() + LauncherMapper.ROOT_LOGGER_LEVEL; + String oozieActionRootLogger = "oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL; + + // check if root log level has already mentioned in action configuration + String rootLogLevel = actionConf.get(oozieActionTypeRootLogger, actionConf.get(oozieActionRootLogger)); + if (rootLogLevel != null) { + // root log level is mentioned in action configuration + return; + } + + // set the root log level which is mentioned in oozie default + rootLogLevel = ConfigurationService.get(oozieActionTypeRootLogger); + if (rootLogLevel != null && rootLogLevel.length() > 0) { + actionConf.set(oozieActionRootLogger, rootLogLevel); + } + else { + rootLogLevel = ConfigurationService.get(oozieActionRootLogger); + if (rootLogLevel != null && rootLogLevel.length() > 0) { + actionConf.set(oozieActionRootLogger, rootLogLevel); + } + } + } + Configuration addToCache(Configuration conf, Path appPath, String filePath, boolean archive) throws ActionExecutorException { @@ -1673,10 +1702,12 @@ public class JavaActionExecutor extends ActionExecutor { } } + @Override public boolean requiresNameNodeJobTracker() { return true; } + @Override public boolean supportsConfigurationJobXML() { return true; } http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index faf3740..ca49fa6 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -1804,6 +1804,14 @@ </description> </property> + <property> + <name>oozie.action.rootlogger.log.level</name> + <value>INFO</value> + <description> + Logging level for root logger + </description> + </property> + <!-- HadoopActionExecutor --> <!-- This is common to the subclasses action executors for map-reduce and pig --> http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 70f33ea..86f0ed2 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -2702,4 +2702,30 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals(4, conf.size()); } } + + public void testSetRootLoggerLevel() throws Exception { + String oozieActionRootLogger = "oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL; + String oozieActionHiveRootLogger = "oozie.action.hive" + LauncherMapper.ROOT_LOGGER_LEVEL; + + // default should be INFO + JavaActionExecutor jae = new JavaActionExecutor(); + Configuration conf = new Configuration(false); + jae.setRootLoggerLevel(conf); + assertEquals("INFO", conf.get(oozieActionRootLogger)); + + // explicitly set root logger level to DEBUG + jae = new JavaActionExecutor(); + conf = new Configuration(false); + conf.set(oozieActionRootLogger, "DEBUG"); + jae.setRootLoggerLevel(conf); + assertEquals("DEBUG", conf.get(oozieActionRootLogger)); + + // explicitly set root logger level to DEBUG for hive action + HiveActionExecutor hae = new HiveActionExecutor(); + conf = new Configuration(false); + conf.set(oozieActionHiveRootLogger, "DEBUG"); + hae.setRootLoggerLevel(conf); + assertEquals("DEBUG", conf.get(oozieActionHiveRootLogger)); + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 391834b..0804939 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2430 Add root logger for hive,sqoop action (satishsaley via puru) OOZIE-2441 SubWorkflow action with propagate-configuration but no global section throws NPE on submit (rkanter) OOZIE-2370 Docs for Coordinator Action Status Notification has wrong property name (eeeva via rkanter) OOZIE-2419 HBase credentials are not correctly proxied (qwertymaniac via rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java index 94e586b..cb0172d 100644 --- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java +++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java @@ -143,7 +143,9 @@ public class HiveMain extends LauncherMain { } String logLevel = hiveConf.get("oozie.hive.log.level", "INFO"); + String rootLogLevel = hiveConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO"); + hadoopProps.setProperty("log4j.rootLogger", rootLogLevel + ", A"); hadoopProps.setProperty("log4j.logger.org.apache.hadoop.hive", logLevel + ", A"); hadoopProps.setProperty("log4j.logger.hive", logLevel + ", A"); hadoopProps.setProperty("log4j.logger.DataNucleus", logLevel + ", A"); @@ -151,13 +153,14 @@ public class HiveMain extends LauncherMain { hadoopProps.setProperty("log4j.logger.JPOX", logLevel + ", A"); hadoopProps.setProperty("log4j.appender.A", "org.apache.log4j.ConsoleAppender"); hadoopProps.setProperty("log4j.appender.A.layout", "org.apache.log4j.PatternLayout"); - hadoopProps.setProperty("log4j.appender.A.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n"); + hadoopProps.setProperty("log4j.appender.A.layout.ConversionPattern", "%-4r [%t] -5p %c %x - %m%n"); hadoopProps.setProperty("log4j.appender.jobid", "org.apache.log4j.FileAppender"); hadoopProps.setProperty("log4j.appender.jobid.file", logFile); hadoopProps.setProperty("log4j.appender.jobid.layout", "org.apache.log4j.PatternLayout"); hadoopProps.setProperty("log4j.appender.jobid.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n"); hadoopProps.setProperty("log4j.logger.org.apache.hadoop.hive.ql.exec", "INFO, jobid"); + hadoopProps.setProperty("log4j.logger.SessionState", "INFO, jobid"); String localProps = new File(HIVE_L4J_PROPS).getAbsolutePath(); OutputStream os1 = new FileOutputStream(localProps); @@ -199,6 +202,7 @@ public class HiveMain extends LauncherMain { return hiveConf; } + @Override protected void run(String[] args) throws Exception { System.out.println(); System.out.println("Oozie Hive action configuration"); http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java index c45073f..545f8e2 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java @@ -83,6 +83,7 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R public static final String HADOOP2_WORKAROUND_DISTRIBUTED_CACHE = "oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache"; public static final String PROPAGATION_CONF_XML = "propagation-conf.xml"; public static final String OOZIE_LAUNCHER_JOB_ID = "oozie.launcher.job.id"; + public static final String ROOT_LOGGER_LEVEL = "rootlogger.log.level"; private void setRecoveryId(Configuration launcherConf, Path actionDir, String recoveryId) throws LauncherException { try { http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java index 8228e88..7ba22f1 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java @@ -165,11 +165,12 @@ public class PigMain extends LauncherMain { if (log4jFile != null) { String pigLogLevel = actionConf.get("oozie.pig.log.level", "INFO"); + String rootLogLevel = actionConf.get("oozie.action ." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO"); // append required PIG properties to the default hadoop log4j file Properties hadoopProps = new Properties(); hadoopProps.load(log4jFile.openStream()); - hadoopProps.setProperty("log4j.rootLogger", pigLogLevel + ", A, B"); + hadoopProps.setProperty("log4j.rootLogger", rootLogLevel + ", A, B"); hadoopProps.setProperty("log4j.logger.org.apache.pig", pigLogLevel + ", A, B"); hadoopProps.setProperty("log4j.additivity.org.apache.pig", "false"); hadoopProps.setProperty("log4j.appender.A", "org.apache.log4j.ConsoleAppender"); http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java index 607ba1b..7ff125d 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java @@ -153,10 +153,12 @@ public class PigMainWithOldAPI extends LauncherMain { if (log4jFile != null) { String pigLogLevel = actionConf.get("oozie.pig.log.level", "INFO"); + String rootLogLevel = actionConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO"); // append required PIG properties to the default hadoop log4j file Properties hadoopProps = new Properties(); hadoopProps.load(log4jFile.openStream()); + hadoopProps.setProperty("log4j.rootLogger", rootLogLevel + ", A, B"); hadoopProps.setProperty("log4j.logger.org.apache.pig", pigLogLevel + ", A, B"); hadoopProps.setProperty("log4j.appender.A", "org.apache.log4j.ConsoleAppender"); hadoopProps.setProperty("log4j.appender.A.layout", "org.apache.log4j.PatternLayout"); http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java ---------------------------------------------------------------------- diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java index 6ba7238..59a0209 100644 --- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java +++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java @@ -120,7 +120,9 @@ public class SqoopMain extends LauncherMain { } String logLevel = sqoopConf.get("oozie.sqoop.log.level", "INFO"); + String rootLogLevel = sqoopConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO"); + hadoopProps.setProperty("log4j.rootLogger", rootLogLevel + ", A"); hadoopProps.setProperty("log4j.logger.org.apache.sqoop", logLevel + ", A"); hadoopProps.setProperty("log4j.appender.A", "org.apache.log4j.ConsoleAppender"); hadoopProps.setProperty("log4j.appender.A.layout", "org.apache.log4j.PatternLayout");
