Repository: oozie Updated Branches: refs/heads/master fafb5cf03 -> e9d3d3cb2
OOZIE-2471 Show child job url tab for distcp Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/e9d3d3cb Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/e9d3d3cb Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/e9d3d3cb Branch: refs/heads/master Commit: e9d3d3cb269ed954d38de0306929e4857d650301 Parents: fafb5cf Author: Purshotam Shah <[email protected]> Authored: Tue May 10 09:48:13 2016 -0700 Committer: Purshotam Shah <[email protected]> Committed: Tue May 10 09:48:13 2016 -0700 ---------------------------------------------------------------------- .../action/hadoop/DistcpActionExecutor.java | 19 ++++++ .../action/hadoop/TestDistCpActionExecutor.java | 18 +++++- .../oozie/action/hadoop/TestDistcpMain.java | 10 +++ release-log.txt | 1 + .../apache/oozie/action/hadoop/DistcpMain.java | 64 +++++++++++++++++++- webapp/src/main/webapp/oozie-console.js | 2 +- 6 files changed, 111 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/e9d3d3cb/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java index 42f2965..96726da 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java @@ -18,16 +18,24 @@ package org.apache.oozie.action.hadoop; +import java.io.IOException; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.action.ActionExecutorException; +import org.apache.oozie.action.ActionExecutor.Context; +import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.service.ConfigurationService; +import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.service.Services; import org.apache.oozie.util.XLog; import org.jdom.Element; +import org.jdom.JDOMException; public class DistcpActionExecutor extends JavaActionExecutor{ public static final String CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS = "org.apache.oozie.action.hadoop.DistcpMain"; @@ -117,4 +125,15 @@ public class DistcpActionExecutor extends JavaActionExecutor{ return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS); } + @Override + protected void getActionData(FileSystem actionFs, RunningJob runningJob, WorkflowAction action, Context context) + throws HadoopAccessorException, JDOMException, IOException, URISyntaxException { + super.getActionData(actionFs, runningJob, action, context); + readExternalChildIDs(action, context); + } + + @Override + protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException { + return true; + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/e9d3d3cb/core/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java index 939a332..e8a140f 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistCpActionExecutor.java @@ -31,10 +31,12 @@ import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.WorkflowActionBean; import org.apache.oozie.WorkflowJobBean; +import org.apache.oozie.action.hadoop.ActionExecutorTestCase.Context; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.service.HadoopAccessorService; import org.apache.oozie.service.Services; import org.apache.oozie.service.WorkflowAppService; +import org.apache.oozie.test.XTestCase.Predicate; import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.XConfiguration; @@ -96,8 +98,21 @@ public class TestDistCpActionExecutor extends ActionExecutorTestCase{ assertEquals(readContent[offset], content[offset]); offset++; } - } + // Check for external ids + DistcpActionExecutor ae = new DistcpActionExecutor(); + WorkflowAction wfAction = context.getAction(); + ae.check(context, wfAction); + ae.end(context, wfAction); + + assertEquals("SUCCEEDED", wfAction.getExternalStatus()); + String externalIds = wfAction.getExternalChildIDs(); + assertNotNull(externalIds); + assertNotSame("", externalIds); + // check for the expected prefix of hadoop jobIDs + assertTrue(externalIds.contains("job_")); + + } protected Context createContext(String actionXml) throws Exception { DistcpActionExecutor ae = new DistcpActionExecutor(); @@ -148,4 +163,5 @@ public class TestDistCpActionExecutor extends ActionExecutorTestCase{ assertNotNull(runningJob); return runningJob; } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/e9d3d3cb/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java index 9581c5f..10c0e1c 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java @@ -58,6 +58,16 @@ public class TestDistcpMain extends MainTestCase { System.setProperty("oozie.action.conf.xml", actionXml.getAbsolutePath()); + File statsDataFile = new File(getTestCaseDir(), "statsdata.properties"); + File hadoopIdsFile = new File(getTestCaseDir(), "hadoopIds"); + File outputDataFile = new File(getTestCaseDir(), "outputdata.properties"); + + setSystemProperty("oozie.launcher.job.id", "" + System.currentTimeMillis()); + setSystemProperty("oozie.action.conf.xml", actionXml.getAbsolutePath()); + setSystemProperty("oozie.action.stats.properties", statsDataFile.getAbsolutePath()); + setSystemProperty("oozie.action.externalChildIDs", hadoopIdsFile.getAbsolutePath()); + setSystemProperty("oozie.action.output.properties", outputDataFile.getAbsolutePath()); + // Check normal execution DistcpMain.main(new String[]{inputDir.toString(), outputDir.toString()}); assertTrue(getFileSystem().exists(outputDir)); http://git-wip-us.apache.org/repos/asf/oozie/blob/e9d3d3cb/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 67f48e8..e9aa963 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2471 Show child job url tab for distcp (satishsaley via puru) OOZIE-2511 SubWorkflow missing variable set from option if config-default is present in parent workflow (asasvari via rkanter) OOZIE-2391 spark-opts value in workflow.xml is not parsed properly (gezapeti via rkanter) OOZIE-2489 XML parsing is vulnerable (fdenes via rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/e9d3d3cb/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java ---------------------------------------------------------------------- diff --git a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java index 325798f..6ac5ad6 100644 --- a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java +++ b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java @@ -18,18 +18,28 @@ package org.apache.oozie.action.hadoop; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.net.URL; +import java.util.Properties; +import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import org.apache.log4j.PropertyConfigurator; public class DistcpMain extends JavaMain { private Constructor<?> construct; private Object[] constArgs; - + private static final String DISTCP_LOG4J_PROPS = "distcp-log4j.properties"; + private static final Pattern[] DISTCP_JOB_IDS_PATTERNS = { Pattern.compile("Job complete: (job_\\S*)"), + Pattern.compile("Job (job_\\S*) completed successfully") }; public static void main(String[] args) throws Exception { run(DistcpMain.class, args); } @@ -39,6 +49,7 @@ public class DistcpMain extends JavaMain { Configuration actionConf = loadActionConf(); LauncherMainHadoopUtils.killChildYarnJobs(actionConf); + String logFile = setUpDistcpLog4J(actionConf); Class<?> klass = actionConf.getClass(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, org.apache.hadoop.tools.DistCp.class); System.out.println("Main class : " + klass.getName()); @@ -69,6 +80,9 @@ public class DistcpMain extends JavaMain { catch (InvocationTargetException ex) { throw new JavaMainException(ex.getCause()); } + finally { + writeExternalChildIDs(logFile, DISTCP_JOB_IDS_PATTERNS, "Distcp"); + } } protected void getConstructorAndArgs(Class<?> klass, Configuration actionConf) throws Exception { @@ -95,4 +109,52 @@ public class DistcpMain extends JavaMain { } } + public static String setUpDistcpLog4J(Configuration distcpConf) throws IOException { + // Logfile to capture job IDs + String hadoopJobId = System.getProperty("oozie.launcher.job.id"); + if (hadoopJobId == null) { + throw new RuntimeException("Launcher Hadoop Job ID system,property not set"); + } + + String logFile = new File("distcp-oozie-" + hadoopJobId + ".log").getAbsolutePath(); + + Properties hadoopProps = new Properties(); + + // Preparing log4j configuration + URL log4jFile = Thread.currentThread().getContextClassLoader().getResource("log4j.properties"); + if (log4jFile != null) { + // getting hadoop log4j configuration + hadoopProps.load(log4jFile.openStream()); + } + + String logLevel = distcpConf.get("oozie.distcp.log.level", "INFO"); + String rootLogLevel = distcpConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO"); + + hadoopProps.setProperty("log4j.rootLogger", rootLogLevel + ", A"); + hadoopProps.setProperty("log4j.logger.org.apache.hadoop.tools", logLevel + ", A, jobid"); + hadoopProps.setProperty("log4j.additivity.org.apache.hadoop.tools", "false"); + hadoopProps.setProperty("log4j.appender.A", "org.apache.log4j.ConsoleAppender"); + hadoopProps.setProperty("log4j.appender.A.layout", "org.apache.log4j.PatternLayout"); + hadoopProps.setProperty("log4j.appender.A.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n"); + + hadoopProps.setProperty("log4j.appender.jobid", "org.apache.log4j.FileAppender"); + hadoopProps.setProperty("log4j.appender.jobid.file", logFile); + hadoopProps.setProperty("log4j.appender.jobid.layout", "org.apache.log4j.PatternLayout"); + hadoopProps.setProperty("log4j.appender.jobid.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n"); + hadoopProps.setProperty("log4j.logger.org.apache.hadoop.mapred", "INFO, jobid"); + hadoopProps.setProperty("log4j.logger.org.apache.hadoop.mapreduce.Job", "INFO, jobid"); + + String localProps = new File(DISTCP_LOG4J_PROPS).getAbsolutePath(); + OutputStream os1 = new FileOutputStream(localProps); + try { + hadoopProps.store(os1, ""); + } + finally { + os1.close(); + } + + PropertyConfigurator.configure(DISTCP_LOG4J_PROPS); + + return logFile; + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/e9d3d3cb/webapp/src/main/webapp/oozie-console.js ---------------------------------------------------------------------- diff --git a/webapp/src/main/webapp/oozie-console.js b/webapp/src/main/webapp/oozie-console.js index 31dcc3d..99dc6ce 100644 --- a/webapp/src/main/webapp/oozie-console.js +++ b/webapp/src/main/webapp/oozie-console.js @@ -765,7 +765,7 @@ function jobDetailsPopup(response, request) { items : urlUnit }; if (actionStatus.type == "pig" || actionStatus.type == "hive" || actionStatus.type == "map-reduce" - || actionStatus.type == "hive2" || actionStatus.type == "sqoop") { + || actionStatus.type == "hive2" || actionStatus.type == "sqoop" || actionStatus.type == "distcp") { var tabPanel = win.items.get(0); tabPanel.add(childJobsItem); }
