Repository: oozie Updated Branches: refs/heads/master 0c90b8c36 -> 42cebf6e2
OOZIE-1964 Hive Server 2 action doesn't return Hadoop Job IDs (rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/42cebf6e Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/42cebf6e Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/42cebf6e Branch: refs/heads/master Commit: 42cebf6e224d07c2bb698a2c10e8c6377898e42b Parents: 0c90b8c Author: Robert Kanter <[email protected]> Authored: Tue Mar 10 15:03:53 2015 -0700 Committer: Robert Kanter <[email protected]> Committed: Tue Mar 10 15:03:53 2015 -0700 ---------------------------------------------------------------------- .../action/hadoop/Hive2ActionExecutor.java | 16 +++++++++++- .../oozie/action/hadoop/HiveActionExecutor.java | 9 +------ .../oozie/action/hadoop/JavaActionExecutor.java | 10 ++++++++ .../action/hadoop/SqoopActionExecutor.java | 9 +------ release-log.txt | 1 + .../apache/oozie/action/hadoop/HiveMain.java | 25 +----------------- .../apache/oozie/action/hadoop/Hive2Main.java | 27 +++++++++++++++++--- .../action/hadoop/TestHive2ActionExecutor.java | 7 +++++ .../oozie/action/hadoop/LauncherMain.java | 27 +++++++++++++++++++- .../org/apache/oozie/action/hadoop/PigMain.java | 5 +--- .../apache/oozie/action/hadoop/SqoopMain.java | 14 +--------- webapp/src/main/webapp/oozie-console.js | 5 ++-- 12 files changed, 91 insertions(+), 64 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java index 8f86c09..704b762 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java @@ -20,13 +20,20 @@ package org.apache.oozie.action.hadoop; import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS; +import java.io.IOException; +import java.io.StringReader; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; +import java.util.Properties; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.WorkflowAction; +import org.apache.oozie.service.HadoopAccessorException; import org.jdom.Element; import org.jdom.JDOMException; import org.jdom.Namespace; @@ -115,7 +122,14 @@ public class Hive2ActionExecutor extends ScriptLanguageActionExecutor { @Override protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException { - return false; + return true; + } + + @Override + protected void getActionData(FileSystem actionFs, RunningJob runningJob, WorkflowAction action, Context context) + throws HadoopAccessorException, JDOMException, IOException, URISyntaxException { + super.getActionData(actionFs, runningJob, action, context); + readExternalChildIDs(action, context); } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java index 701d1c1..dbb50ba 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java @@ -115,14 +115,7 @@ public class HiveActionExecutor extends ScriptLanguageActionExecutor { protected void getActionData(FileSystem actionFs, RunningJob runningJob, WorkflowAction action, Context context) throws HadoopAccessorException, JDOMException, IOException, URISyntaxException { super.getActionData(actionFs, runningJob, action, context); - - if (action.getData() != null) { - // Load stored Hadoop jobs ids and promote them as external child - // ids on job success - Properties props = new Properties(); - props.load(new StringReader(action.getData())); - context.setExternalChildIDs((String) props.get(LauncherMain.HADOOP_JOBS)); - } + readExternalChildIDs(action, context); } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 5f8646d..cf22ed6 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -1436,6 +1436,16 @@ public class JavaActionExecutor extends ActionExecutor { throws HadoopAccessorException, JDOMException, IOException, URISyntaxException { } + protected final void readExternalChildIDs(WorkflowAction action, Context context) throws IOException { + if (action.getData() != null) { + // Load stored Hadoop jobs ids and promote them as external child ids + // See LauncherMain#writeExternalChildIDs for how they are written + Properties props = new Properties(); + props.load(new StringReader(action.getData())); + context.setExternalChildIDs((String) props.get(LauncherMain.HADOOP_JOBS)); + } + } + protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException { Element eConf = XmlUtils.parseXml(action.getConf()); Namespace ns = eConf.getNamespace(); http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java index af77c33..6813a37 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java @@ -243,14 +243,7 @@ public class SqoopActionExecutor extends JavaActionExecutor { protected void getActionData(FileSystem actionFs, RunningJob runningJob, WorkflowAction action, Context context) throws HadoopAccessorException, JDOMException, IOException, URISyntaxException{ super.getActionData(actionFs, runningJob, action, context); - - if (action.getData() != null) { - // Load stored Hadoop jobs ids and promote them as external child - // ids - Properties props = new Properties(); - props.load(new StringReader(action.getData())); - context.setExternalChildIDs((String) props.get(LauncherMain.HADOOP_JOBS)); - } + readExternalChildIDs(action, context); } @Override http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 6ab5737..9bd3af2 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.2.0 release (trunk - unreleased) +OOZIE-1964 Hive Server 2 action doesn't return Hadoop Job IDs (rkanter) OOZIE-2126 SSH action can be too fast for Oozie sometimes (rkanter) OOZIE-2142 Changing the JT whitelist causes running Workflows to stay RUNNING forever (rkanter) OOZIE-2164 make master parameterizable in Spark action example (wypoon via rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java index df9828a..5ea4e1a 100644 --- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java +++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java @@ -296,30 +296,7 @@ public class HiveMain extends LauncherMain { } finally { System.out.println("\n<<< Invocation of Hive command completed <<<\n"); - writeExternalChildIDs(logFile); - - } - } - - private void writeExternalChildIDs(String logFile) { - // harvesting and recording Hadoop Job IDs - try { - Properties jobIds = getHadoopJobIds(logFile, HIVE_JOB_IDS_PATTERNS); - File file = new File(System.getProperty(LauncherMapper.ACTION_PREFIX - + LauncherMapper.ACTION_DATA_OUTPUT_PROPS)); - OutputStream os = new FileOutputStream(file); - try { - jobIds.store(os, ""); - } - finally { - os.close(); - } - System.out.println(" Hadoop Job IDs executed by Hive: " + jobIds.getProperty(HADOOP_JOBS)); - System.out.println(); - } - catch (Exception e) { - System.out.println("WARN: Error getting Hadoop Job IDs executed by Hive"); - e.printStackTrace(System.out); + writeExternalChildIDs(logFile, HIVE_JOB_IDS_PATTERNS, "Hive"); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java ---------------------------------------------------------------------- diff --git a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java index 6d4f44a..ca9fcb4 100644 --- a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java +++ b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java @@ -20,18 +20,25 @@ package org.apache.oozie.action.hadoop; import java.io.BufferedReader; import java.io.File; +import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; +import java.io.PrintStream; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.regex.Pattern; +import org.apache.commons.io.output.TeeOutputStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hive.beeline.BeeLine; public class Hive2Main extends LauncherMain { + private static final Pattern[] HIVE2_JOB_IDS_PATTERNS = { + Pattern.compile("Ended Job = (job_\\S*)") + }; private static final Set<String> DISALLOWED_BEELINE_OPTIONS = new HashSet<String>(); static { @@ -93,6 +100,13 @@ public class Hive2Main extends LauncherMain { Configuration actionConf = initActionConf(); + //Logfile to capture job IDs + String hadoopJobId = System.getProperty("oozie.launcher.job.id"); + if (hadoopJobId == null) { + throw new RuntimeException("Launcher Hadoop Job ID system property not set"); + } + String logFile = new File("hive2-oozie-" + hadoopJobId + ".log").getAbsolutePath(); + List<String> arguments = new ArrayList<String>(); String jdbcUrl = actionConf.get(Hive2ActionExecutor.HIVE2_JDBC_URL); if (jdbcUrl == null) { @@ -199,7 +213,7 @@ public class Hive2Main extends LauncherMain { System.out.flush(); try { - runBeeline(arguments.toArray(new String[arguments.size()])); + runBeeline(arguments.toArray(new String[arguments.size()]), logFile); } catch (SecurityException ex) { if (LauncherSecurityManager.getExitInvoked()) { @@ -210,11 +224,18 @@ public class Hive2Main extends LauncherMain { } finally { System.out.println("\n<<< Invocation of Beeline command completed <<<\n"); + writeExternalChildIDs(logFile, HIVE2_JOB_IDS_PATTERNS, "Beeline"); } } - private void runBeeline(String[] args) throws Exception { - BeeLine.main(args); + private void runBeeline(String[] args, String logFile) throws Exception { + // We do this instead of calling BeeLine.main so we can duplicate the error stream for harvesting Hadoop child job IDs + BeeLine beeLine = new BeeLine(); + beeLine.setErrorStream(new PrintStream(new TeeOutputStream(System.err, new FileOutputStream(logFile)))); + int status = beeLine.begin(args, null); + if (status != 0) { + System.exit(status); + } } private static String readStringFromFile(String filePath) throws IOException { http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java index ac39b94..16d0267 100644 --- a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java +++ b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java @@ -25,6 +25,7 @@ import java.text.MessageFormat; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Properties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -174,6 +175,12 @@ public class TestHive2ActionExecutor extends ActionExecutorTestCase { ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); + assertNotNull(context.getAction().getData()); + Properties outputData = new Properties(); + outputData.load(new StringReader(context.getAction().getData())); + assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS)); + assertEquals(outputData.get(LauncherMain.HADOOP_JOBS), context.getExternalChildIDs()); + assertTrue(fs.exists(outputDir)); assertTrue(fs.isDirectory(outputDir)); } http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java index 552c862..0860484 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java @@ -20,8 +20,10 @@ package org.apache.oozie.action.hadoop; import java.io.BufferedReader; import java.io.File; +import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; +import java.io.OutputStream; import java.io.StringWriter; import java.util.Collection; import java.util.Map; @@ -44,7 +46,7 @@ public abstract class LauncherMain { main.run(args); } - public static Properties getHadoopJobIds(String logFile, Pattern[] patterns) throws IOException { + protected static Properties getHadoopJobIds(String logFile, Pattern[] patterns) throws IOException { Properties props = new Properties(); StringBuffer sb = new StringBuffer(100); if (!new File(logFile).exists()) { @@ -76,6 +78,29 @@ public abstract class LauncherMain { return props; } + protected static void writeExternalChildIDs(String logFile, Pattern[] patterns, String name) { + // Harvesting and recording Hadoop Job IDs + // See JavaActionExecutor#readExternalChildIDs for how they are read + try { + Properties jobIds = getHadoopJobIds(logFile, patterns); + File file = new File(System.getProperty(LauncherMapper.ACTION_PREFIX + + LauncherMapper.ACTION_DATA_OUTPUT_PROPS)); + OutputStream os = new FileOutputStream(file); + try { + jobIds.store(os, ""); + } + finally { + os.close(); + } + System.out.println(" Hadoop Job IDs executed by " + name + ": " + jobIds.getProperty(HADOOP_JOBS)); + System.out.println(); + } + catch (Exception e) { + System.out.println("WARN: Error getting Hadoop Job IDs executed by " + name); + e.printStackTrace(System.out); + } + } + protected abstract void run(String[] args) throws Exception; /** http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java index d924018..129022a 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java @@ -235,10 +235,7 @@ public class PigMain extends LauncherMain { // So retrieving hadoop Ids here File file = new File(System.getProperty(EXTERNAL_CHILD_IDS)); if (!file.exists()) { - Properties props = getHadoopJobIds(logFile, PIG_JOB_IDS_PATTERNS); - writeExternalData(props.getProperty(HADOOP_JOBS), file); - System.out.println(" Hadoop Job IDs executed by Pig: " + props.getProperty(HADOOP_JOBS)); - System.out.println(); + writeExternalChildIDs(logFile, PIG_JOB_IDS_PATTERNS, "Pig"); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java ---------------------------------------------------------------------- diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java index d672bf3..1ffaf10 100644 --- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java +++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java @@ -188,19 +188,7 @@ public class SqoopMain extends LauncherMain { System.out.println(); // harvesting and recording Hadoop Job IDs - Properties jobIds = getHadoopJobIds(logFile, SQOOP_JOB_IDS_PATTERNS); - - File file = new File(System.getProperty(LauncherMapper.ACTION_PREFIX - + LauncherMapper.ACTION_DATA_OUTPUT_PROPS)); - OutputStream os = new FileOutputStream(file); - try { - jobIds.store(os, ""); - } - finally { - os.close(); - } - System.out.println(" Hadoop Job IDs executed by Sqoop: " + jobIds.getProperty(HADOOP_JOBS)); - System.out.println(); + writeExternalChildIDs(logFile, SQOOP_JOB_IDS_PATTERNS, "Sqoop"); } protected void runSqoopJob(String[] args) throws Exception { http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/webapp/src/main/webapp/oozie-console.js ---------------------------------------------------------------------- diff --git a/webapp/src/main/webapp/oozie-console.js b/webapp/src/main/webapp/oozie-console.js index e2b37a6..e07680f 100644 --- a/webapp/src/main/webapp/oozie-console.js +++ b/webapp/src/main/webapp/oozie-console.js @@ -728,7 +728,7 @@ function jobDetailsPopup(response, request) { })] }); - // Tab to show list of child Job URLs for pig action + // Tab to show list of child Job URLs var childJobsItem = { title : 'Child Job URLs', autoScroll : true, @@ -739,7 +739,8 @@ function jobDetailsPopup(response, request) { width: 540, items : urlUnit }; - if (actionStatus.type == "pig" || actionStatus.type == "hive" || actionStatus.type == "map-reduce") { + if (actionStatus.type == "pig" || actionStatus.type == "hive" || actionStatus.type == "map-reduce" + || actionStatus.type == "hive2" || actionStatus.type == "sqoop") { var tabPanel = win.items.get(0); tabPanel.add(childJobsItem); }
