OOZIE-2582 Populating external child Ids for action failures (abhishekbafna via rohini)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/e8a9b24b Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/e8a9b24b Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/e8a9b24b Branch: refs/heads/oya Commit: e8a9b24b9916df20cfbc1c48e5e1a755151d8245 Parents: 76b60cc Author: Rohini Palaniswamy <roh...@apache.org> Authored: Tue Sep 27 12:05:19 2016 -0700 Committer: Rohini Palaniswamy <roh...@apache.org> Committed: Tue Sep 27 12:05:19 2016 -0700 ---------------------------------------------------------------------- .../action/hadoop/DistcpActionExecutor.java | 27 +------ .../action/hadoop/Hive2ActionExecutor.java | 29 ++----- .../oozie/action/hadoop/HiveActionExecutor.java | 29 ++----- .../oozie/action/hadoop/JavaActionExecutor.java | 21 +++--- .../action/hadoop/SparkActionExecutor.java | 25 +------ .../action/hadoop/SqoopActionExecutor.java | 41 ++-------- .../oozie/action/hadoop/TestDistcpMain.java | 2 + examples/src/main/apps/hive/script.q | 1 + release-log.txt | 1 + .../apache/oozie/action/hadoop/DistcpMain.java | 9 ++- .../apache/oozie/action/hadoop/HiveMain.java | 5 +- .../action/hadoop/TestHiveActionExecutor.java | 15 +--- .../oozie/action/hadoop/TestHiveMain.java | 8 +- .../apache/oozie/action/hadoop/Hive2Main.java | 3 +- .../action/hadoop/TestHive2ActionExecutor.java | 57 +++++++++++--- .../oozie/action/hadoop/LauncherMain.java | 79 ++++++++++---------- .../apache/oozie/action/hadoop/ShellMain.java | 2 +- .../org/apache/oozie/action/hadoop/PigMain.java | 8 +- .../oozie/action/hadoop/PigMainWithOldAPI.java | 2 +- .../apache/oozie/action/hadoop/SparkMain.java | 1 + .../apache/oozie/action/hadoop/SqoopMain.java | 16 ++-- .../action/hadoop/TestSqoopActionExecutor.java | 22 +----- 22 files changed, 158 insertions(+), 245 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java index 96726da..78cd257 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java @@ -18,24 +18,15 @@ package org.apache.oozie.action.hadoop; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; - import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.action.ActionExecutorException; -import org.apache.oozie.action.ActionExecutor.Context; -import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.service.ConfigurationService; -import org.apache.oozie.service.HadoopAccessorException; -import org.apache.oozie.service.Services; import org.apache.oozie.util.XLog; import org.jdom.Element; -import org.jdom.JDOMException; + +import java.util.ArrayList; +import java.util.List; public class DistcpActionExecutor extends JavaActionExecutor{ public static final String CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS = "org.apache.oozie.action.hadoop.DistcpMain"; @@ -124,16 +115,4 @@ public class DistcpActionExecutor extends JavaActionExecutor{ protected String getLauncherMain(Configuration launcherConf, Element actionXml) { return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS); } - - @Override - protected void getActionData(FileSystem actionFs, RunningJob runningJob, WorkflowAction action, Context context) - throws HadoopAccessorException, JDOMException, IOException, URISyntaxException { - super.getActionData(actionFs, runningJob, action, context); - readExternalChildIDs(action, context); - } - - @Override - protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException { - return true; - } } http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java index b5b1bf9..2aed936 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java @@ -18,24 +18,17 @@ package org.apache.oozie.action.hadoop; -import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; - import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.action.ActionExecutorException; -import org.apache.oozie.client.WorkflowAction; -import org.apache.oozie.service.HadoopAccessorException; import org.jdom.Element; -import org.jdom.JDOMException; import org.jdom.Namespace; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS; + public class Hive2ActionExecutor extends ScriptLanguageActionExecutor { private static final String HIVE2_MAIN_CLASS_NAME = "org.apache.oozie.action.hadoop.Hive2Main"; @@ -128,18 +121,6 @@ public class Hive2ActionExecutor extends ScriptLanguageActionExecutor { return conf; } - @Override - protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException { - return true; - } - - @Override - protected void getActionData(FileSystem actionFs, RunningJob runningJob, WorkflowAction action, Context context) - throws HadoopAccessorException, JDOMException, IOException, URISyntaxException { - super.getActionData(actionFs, runningJob, action, context); - readExternalChildIDs(action, context); - } - /** * Return the sharelib name for the action. * http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java index c74e9e6..8e2453e 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java @@ -18,28 +18,21 @@ package org.apache.oozie.action.hadoop; -import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; - import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.action.ActionExecutorException; -import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.client.XOozieClient; import org.apache.oozie.service.ConfigurationService; -import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.service.HadoopAccessorService; import org.jdom.Element; -import org.jdom.JDOMException; import org.jdom.Namespace; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS; + public class HiveActionExecutor extends ScriptLanguageActionExecutor { private static final String HIVE_MAIN_CLASS_NAME = "org.apache.oozie.action.hadoop.HiveMain"; @@ -119,18 +112,6 @@ public class HiveActionExecutor extends ScriptLanguageActionExecutor { return conf; } - @Override - protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException { - return true; - } - - @Override - protected void getActionData(FileSystem actionFs, RunningJob runningJob, WorkflowAction action, Context context) - throws HadoopAccessorException, JDOMException, IOException, URISyntaxException { - super.getActionData(actionFs, runningJob, action, context); - readExternalChildIDs(action, context); - } - /** * Return the sharelib name for the action. * http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index ad07b57..65996d9 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -1465,6 +1465,17 @@ public class JavaActionExecutor extends ActionExecutor { context.setExternalChildIDs(externalIDs); LOG.info(XLog.STD, "Hadoop Jobs launched : [{0}]", externalIDs); } + else if (LauncherMapperHelper.hasOutputData(actionData)) { + // Load stored Hadoop jobs ids and promote them as external child ids + // This is for jobs launched with older release during upgrade to Oozie 4.3 + Properties props = PropertiesUtils.stringToProperties(actionData + .get(LauncherMapper.ACTION_DATA_OUTPUT_PROPS)); + if (props.get(LauncherMain.HADOOP_JOBS) != null) { + externalIDs = (String) props.get(LauncherMain.HADOOP_JOBS); + context.setExternalChildIDs(externalIDs); + LOG.info(XLog.STD, "Hadoop Jobs launched : [{0}]", externalIDs); + } + } } if (runningJob.isComplete()) { // fetching action output and stats for the Map-Reduce action. @@ -1566,16 +1577,6 @@ public class JavaActionExecutor extends ActionExecutor { throws HadoopAccessorException, JDOMException, IOException, URISyntaxException { } - protected final void readExternalChildIDs(WorkflowAction action, Context context) throws IOException { - if (action.getData() != null) { - // Load stored Hadoop jobs ids and promote them as external child ids - // See LauncherMain#writeExternalChildIDs for how they are written - Properties props = new Properties(); - props.load(new StringReader(action.getData())); - context.setExternalChildIDs((String) props.get(LauncherMain.HADOOP_JOBS)); - } - } - protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException { Element eConf = XmlUtils.parseXml(action.getConf()); Namespace ns = eConf.getNamespace(); http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java index 15a641b..8e8d7d3 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java @@ -18,27 +18,22 @@ package org.apache.oozie.action.hadoop; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.service.ConfigurationService; -import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.service.Services; import org.apache.oozie.service.SparkConfigurationService; import org.jdom.Element; -import org.jdom.JDOMException; import org.jdom.Namespace; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + public class SparkActionExecutor extends JavaActionExecutor { public static final String SPARK_MAIN_CLASS_NAME = "org.apache.oozie.action.hadoop.SparkMain"; public static final String TASK_USER_PRECEDENCE = "mapreduce.task.classpath.user.precedence"; // hadoop-2 @@ -165,16 +160,4 @@ public class SparkActionExecutor extends JavaActionExecutor { protected String getLauncherMain(Configuration launcherConf, Element actionXml) { return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, SPARK_MAIN_CLASS_NAME); } - - @Override - protected void getActionData(FileSystem actionFs, RunningJob runningJob, WorkflowAction action, Context context) - throws HadoopAccessorException, JDOMException, IOException, URISyntaxException { - super.getActionData(actionFs, runningJob, action, context); - readExternalChildIDs(action, context); - } - - @Override - protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException { - return true; - } } http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java index 6813a37..6cee32a 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java @@ -18,16 +18,7 @@ package org.apache.oozie.action.hadoop; -import java.io.IOException; -import java.io.StringReader; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; -import java.util.StringTokenizer; - import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.Counters; import org.apache.hadoop.mapred.JobClient; @@ -36,14 +27,18 @@ import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.WorkflowAction; -import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.util.XConfiguration; -import org.apache.oozie.util.XmlUtils; import org.apache.oozie.util.XLog; +import org.apache.oozie.util.XmlUtils; import org.jdom.Element; -import org.jdom.JDOMException; import org.jdom.Namespace; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; +import java.util.StringTokenizer; + public class SqoopActionExecutor extends JavaActionExecutor { public static final String OOZIE_ACTION_EXTERNAL_STATS_WRITE = "oozie.action.external.stats.write"; @@ -231,28 +226,6 @@ public class SqoopActionExecutor extends JavaActionExecutor { } /** - * Get the stats and external child IDs - * - * @param actionFs the FileSystem object - * @param runningJob the runningJob - * @param action the Workflow action - * @param context executor context - * - */ - @Override - protected void getActionData(FileSystem actionFs, RunningJob runningJob, WorkflowAction action, Context context) - throws HadoopAccessorException, JDOMException, IOException, URISyntaxException{ - super.getActionData(actionFs, runningJob, action, context); - readExternalChildIDs(action, context); - } - - @Override - protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException { - return true; - } - - - /** * Return the sharelib name for the action. * * @return returns <code>sqoop</code>. http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java index 098995e..bf08ccb 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java @@ -71,6 +71,8 @@ public class TestDistcpMain extends MainTestCase { // Check normal execution DistcpMain.main(new String[]{inputDir.toString(), outputDir.toString()}); assertTrue(getFileSystem().exists(outputDir)); + assertTrue(hadoopIdsFile.exists()); + assertNotNull(LauncherMapper.getLocalFileContentStr(hadoopIdsFile, "", -1)); fs.delete(outputDir,true); // Check exception handling http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/examples/src/main/apps/hive/script.q ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/hive/script.q b/examples/src/main/apps/hive/script.q index 3abc757..37d6564 100644 --- a/examples/src/main/apps/hive/script.q +++ b/examples/src/main/apps/hive/script.q @@ -15,5 +15,6 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- +DROP TABLE IF EXISTS test; CREATE EXTERNAL TABLE test (a INT) STORED AS TEXTFILE LOCATION '${INPUT}'; INSERT OVERWRITE DIRECTORY '${OUTPUT}' SELECT * FROM test; http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index ef11bb3..10a183a 100644 --- a/release-log.txt +++ b/release-log.txt @@ -3,6 +3,7 @@ -- Oozie 4.3.0 release +OOZIE-2582 Populating external child Ids for action failures (abhishekbafna via rohini) OOZIE-2678 Oozie job -kill doesn't work with tez jobs (abhishekbafna via rohini) OOZIE-2676 Make hadoop-2 as the default profile (gezapeti via rkanter) OOZIE-2487 Temporary workaround for Java 8 Javadoc errors (abhishekbafna via rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java ---------------------------------------------------------------------- diff --git a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java index 6ac5ad6..2b84f92 100644 --- a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java +++ b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java @@ -38,8 +38,11 @@ public class DistcpMain extends JavaMain { private Constructor<?> construct; private Object[] constArgs; private static final String DISTCP_LOG4J_PROPS = "distcp-log4j.properties"; - private static final Pattern[] DISTCP_JOB_IDS_PATTERNS = { Pattern.compile("Job complete: (job_\\S*)"), - Pattern.compile("Job (job_\\S*) completed successfully") }; + private static final Pattern[] DISTCP_JOB_IDS_PATTERNS = { + Pattern.compile("Job complete: (job_\\S*)"), + Pattern.compile("Job (job_\\S*) completed successfully"), + Pattern.compile("Submitted application (application[0-9_]*)") + }; public static void main(String[] args) throws Exception { run(DistcpMain.class, args); } @@ -81,6 +84,7 @@ public class DistcpMain extends JavaMain { throw new JavaMainException(ex.getCause()); } finally { + System.out.println("\n<<< Invocation of DistCp command completed <<<\n"); writeExternalChildIDs(logFile, DISTCP_JOB_IDS_PATTERNS, "Distcp"); } } @@ -143,6 +147,7 @@ public class DistcpMain extends JavaMain { hadoopProps.setProperty("log4j.appender.jobid.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n"); hadoopProps.setProperty("log4j.logger.org.apache.hadoop.mapred", "INFO, jobid"); hadoopProps.setProperty("log4j.logger.org.apache.hadoop.mapreduce.Job", "INFO, jobid"); + hadoopProps.setProperty("log4j.logger.org.apache.hadoop.yarn.client.api.impl.YarnClientImpl", "INFO, jobid"); String localProps = new File(DISTCP_LOG4J_PROPS).getAbsolutePath(); OutputStream os1 = new FileOutputStream(localProps); http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java index 5866117..bbcaefa 100644 --- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java +++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java @@ -42,8 +42,8 @@ import org.apache.hadoop.hive.conf.HiveConf; public class HiveMain extends LauncherMain { private static final Pattern[] HIVE_JOB_IDS_PATTERNS = { - Pattern.compile("Ended Job = (job_\\S*)"), - Pattern.compile("Executing on YARN cluster with App id (application[0-9_]*)") + Pattern.compile("Ended Job = (job_\\S*)"), + Pattern.compile("Submitted application (application[0-9_]*)") }; private static final Set<String> DISALLOWED_HIVE_OPTIONS = new HashSet<String>(); @@ -167,6 +167,7 @@ public class HiveMain extends LauncherMain { hadoopProps.setProperty("log4j.appender.jobid.layout.ConversionPattern", "%d [%t] %-5p %c %x - %m%n"); hadoopProps.setProperty("log4j.logger.org.apache.hadoop.hive.ql.exec", "INFO, jobid"); hadoopProps.setProperty("log4j.logger.SessionState", "INFO, jobid"); + hadoopProps.setProperty("log4j.logger.org.apache.hadoop.yarn.client.api.impl.YarnClientImpl", "INFO, jobid"); String localProps = new File(HIVE_L4J_PROPS).getAbsolutePath(); OutputStream os1 = new FileOutputStream(localProps); http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java index b966d4b..12e1e91 100644 --- a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java +++ b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java @@ -27,7 +27,6 @@ import java.io.Writer; import java.text.MessageFormat; import java.util.Arrays; import java.util.Map; -import java.util.Properties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -181,14 +180,9 @@ public class TestHiveActionExecutor extends ActionExecutorTestCase { ae.check(context, context.getAction()); assertTrue(launcherId.equals(context.getAction().getExternalId())); assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); - assertNotNull(context.getAction().getData()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); - assertNotNull(context.getAction().getData()); - Properties outputData = new Properties(); - outputData.load(new StringReader(context.getAction().getData())); - assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS)); - assertEquals(outputData.get(LauncherMain.HADOOP_JOBS), context.getExternalChildIDs()); + assertNotNull(context.getExternalChildIDs()); //while this works in a real cluster, it does not with miniMR //assertTrue(outputData.getProperty(LauncherMain.HADOOP_JOBS).trim().length() > 0); //assertTrue(!actionData.get(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS).isEmpty()); @@ -215,14 +209,9 @@ public class TestHiveActionExecutor extends ActionExecutorTestCase { ae.check(context, context.getAction()); assertTrue(launcherId.equals(context.getAction().getExternalId())); assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); - assertNotNull(context.getAction().getData()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); - assertNotNull(context.getAction().getData()); - Properties outputData = new Properties(); - outputData.load(new StringReader(context.getAction().getData())); - assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS)); - assertEquals(outputData.get(LauncherMain.HADOOP_JOBS), context.getExternalChildIDs()); + assertNotNull(context.getAction().getExternalChildIDs()); //while this works in a real cluster, it does not with miniMR //assertTrue(outputData.getProperty(LauncherMain.HADOOP_JOBS).trim().length() > 0); //assertTrue(!actionData.get(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS).isEmpty()); http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java index d72e298..879ae2d 100644 --- a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java +++ b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java @@ -121,11 +121,11 @@ public class TestHiveMain extends MainTestCase { assertEquals(props.getProperty("oozie.hive.args.size"), "1"); File hiveSite = new File(classPathDir, "hive-site.xml"); - File outputDataFile = new File(getTestCaseDir(), "outputdata.properties"); + File externalChildIdsFile = new File(getTestCaseDir(), "externalChildIDs"); setSystemProperty("oozie.launcher.job.id", "" + System.currentTimeMillis()); setSystemProperty("oozie.action.conf.xml", actionXml.getAbsolutePath()); - setSystemProperty("oozie.action.output.properties", outputDataFile.getAbsolutePath()); + setSystemProperty("oozie.action.externalChildIDs", externalChildIdsFile.getAbsolutePath()); new LauncherSecurityManager(); String user = System.getProperty("user.name"); @@ -155,8 +155,8 @@ public class TestHiveMain extends MainTestCase { MiniHCatServer.resetHiveConfStaticVariables(); } - assertTrue(outputDataFile.exists()); - assertNotNull(LauncherMapper.getLocalFileContentStr(outputDataFile, "", -1)); + assertTrue(externalChildIdsFile.exists()); + assertNotNull(LauncherMapper.getLocalFileContentStr(externalChildIdsFile, "", -1)); //TODO: I cannot figure out why when log file is not created in this testcase, it works when running in Launcher // Properties props = new Properties(); http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java ---------------------------------------------------------------------- diff --git a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java index e122608..54f7039 100644 --- a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java +++ b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java @@ -39,7 +39,8 @@ import org.apache.hive.beeline.BeeLine; public class Hive2Main extends LauncherMain { private static final Pattern[] HIVE2_JOB_IDS_PATTERNS = { - Pattern.compile("Ended Job = (job_\\S*)") + Pattern.compile("Ended Job = (job_\\S*)"), + Pattern.compile("Submitted application (application[0-9_]*)") }; private static final Set<String> DISALLOWED_BEELINE_OPTIONS = new HashSet<String>(); http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java index 5963e42..b023b79 100644 --- a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java +++ b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java @@ -25,7 +25,6 @@ import java.text.MessageFormat; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Properties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -226,11 +225,8 @@ public class TestHive2ActionExecutor extends ActionExecutorTestCase { assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); - assertNotNull(context.getAction().getData()); - Properties outputData = new Properties(); - outputData.load(new StringReader(context.getAction().getData())); - assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS)); - assertEquals(outputData.get(LauncherMain.HADOOP_JOBS), context.getExternalChildIDs()); + // Disabled external child id check until Hive version is upgraded to 0.14+ + //assertNotNull(context.getExternalChildIDs()); assertTrue(fs.exists(outputDir)); assertTrue(fs.isDirectory(outputDir)); } @@ -265,14 +261,53 @@ public class TestHive2ActionExecutor extends ActionExecutorTestCase { assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); - assertNotNull(context.getAction().getData()); - Properties outputData = new Properties(); - outputData.load(new StringReader(context.getAction().getData())); - assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS)); - assertEquals(outputData.get(LauncherMain.HADOOP_JOBS), context.getExternalChildIDs()); + // Disabled external child id check until Hive version is upgraded to 0.14+ + //assertNotNull(context.getExternalChildIDs()); assertTrue(fs.exists(outputDir)); assertTrue(fs.isDirectory(outputDir)); } + // Negative testcase with incorrect hive-query. + { + String query = getHive2BadScript(inputDir.toString(), outputDir.toString()); + Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME))); + dataWriter.write(SAMPLE_DATA_TEXT); + dataWriter.close(); + Context context = createContext(getQueryActionXml(query)); + final RunningJob launcherJob = submitAction(context, Namespace.getNamespace("uri:oozie:hive2-action:0.2")); + String launcherId = context.getAction().getExternalId(); + waitFor(200 * 1000, new Predicate() { + @Override + public boolean evaluate() throws Exception { + return launcherJob.isComplete(); + } + }); + assertTrue(launcherJob.isSuccessful()); + Configuration conf = new XConfiguration(); + conf.set("user.name", getTestUser()); + Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), + conf); + assertFalse(LauncherMapperHelper.hasIdSwap(actionData)); + Hive2ActionExecutor ae = new Hive2ActionExecutor(); + ae.check(context, context.getAction()); + assertTrue(launcherId.equals(context.getAction().getExternalId())); + assertEquals("FAILED/KILLED", context.getAction().getExternalStatus()); + ae.end(context, context.getAction()); + assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus()); + assertNull(context.getExternalChildIDs()); + } + } + + private String getHive2BadScript(String inputPath, String outputPath) { + StringBuilder buffer = new StringBuilder(NEW_LINE); + buffer.append("set -v;").append(NEW_LINE); + buffer.append("DROP TABLE IF EXISTS test;").append(NEW_LINE); + buffer.append("CREATE EXTERNAL TABLE test (a INT) STORED AS"); + buffer.append(NEW_LINE).append("TEXTFILE LOCATION '"); + buffer.append(inputPath).append("';").append(NEW_LINE); + buffer.append("INSERT OVERWRITE DIRECTORY '"); + buffer.append(outputPath).append("'").append(NEW_LINE); + buffer.append("SELECT (a-1) FROM test-bad;").append(NEW_LINE); + return buffer.toString(); } private RunningJob submitAction(Context context, Namespace ns) throws Exception { http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java index 7c5713c..815f60b 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java @@ -29,8 +29,10 @@ import java.io.OutputStream; import java.io.StringWriter; import java.util.Arrays; import java.util.Collection; +import java.util.LinkedHashSet; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -42,6 +44,11 @@ import org.apache.hadoop.mapred.JobConf; public abstract class LauncherMain { + public static final String ACTION_PREFIX = "oozie.action."; + public static final String EXTERNAL_CHILD_IDS = ACTION_PREFIX + "externalChildIDs"; + public static final String EXTERNAL_ACTION_STATS = ACTION_PREFIX + "stats.properties"; + public static final String EXTERNAL_STATS_WRITE = ACTION_PREFIX + "external.stats.write"; + public static final String OUTPUT_PROPERTIES = ACTION_PREFIX + "output.properties"; public static final String HADOOP_JOBS = "hadoopJobs"; public static final String MAPREDUCE_JOB_TAGS = "mapreduce.job.tags"; public static final String TEZ_APPLICATION_TAGS = "tez.application.tags"; @@ -53,58 +60,52 @@ public abstract class LauncherMain { main.run(args); } - protected static Properties getHadoopJobIds(String logFile, Pattern[] patterns) throws IOException { - Properties props = new Properties(); - StringBuffer sb = new StringBuffer(100); + protected static String getHadoopJobIds(String logFile, Pattern[] patterns) { + Set<String> jobIds = new LinkedHashSet<String>(); if (!new File(logFile).exists()) { - System.err.println("Log file: " + logFile + " not present. Therefore no Hadoop jobids found"); - props.setProperty(HADOOP_JOBS, ""); + System.err.println("Log file: " + logFile + " not present. Therefore no Hadoop job IDs found."); } else { - BufferedReader br = new BufferedReader(new FileReader(logFile)); - String line = br.readLine(); - String separator = ""; - while (line != null) { - for (Pattern pattern : patterns) { - Matcher matcher = pattern.matcher(line); - if (matcher.find()) { - String jobId = matcher.group(1); - if (StringUtils.isEmpty(jobId) || jobId.equalsIgnoreCase("NULL")) { - continue; + try (BufferedReader br = new BufferedReader(new FileReader(logFile))) { + String line = br.readLine(); + while (line != null) { + for (Pattern pattern : patterns) { + Matcher matcher = pattern.matcher(line); + if (matcher.find()) { + String jobId = matcher.group(1); + if (StringUtils.isEmpty(jobId) || jobId.equalsIgnoreCase("NULL")) { + continue; + } + jobId = jobId.replaceAll("application", "job"); + jobIds.add(jobId); } - jobId = jobId.replaceAll("application","job"); - sb.append(separator).append(jobId); - separator = ","; } + line = br.readLine(); } - line = br.readLine(); + } catch (IOException e) { + System.out.println("WARN: Error getting Hadoop Job IDs. logFile: " + logFile); + e.printStackTrace(System.out); } - br.close(); - props.setProperty(HADOOP_JOBS, sb.toString()); } - return props; + return jobIds.isEmpty() ? null : StringUtils.join(jobIds, ","); } protected static void writeExternalChildIDs(String logFile, Pattern[] patterns, String name) { // Harvesting and recording Hadoop Job IDs - // See JavaActionExecutor#readExternalChildIDs for how they are read - try { - Properties jobIds = getHadoopJobIds(logFile, patterns); - File file = new File(System.getProperty(LauncherMapper.ACTION_PREFIX - + LauncherMapper.ACTION_DATA_OUTPUT_PROPS)); - OutputStream os = new FileOutputStream(file); - try { - jobIds.store(os, ""); + String jobIds = getHadoopJobIds(logFile, patterns); + if (jobIds != null) { + File externalChildIdsFile = new File(System.getProperty(EXTERNAL_CHILD_IDS)); + try (OutputStream externalChildIdsStream = new FileOutputStream(externalChildIdsFile)) { + externalChildIdsStream.write(jobIds.getBytes()); + System.out.println("Hadoop Job IDs executed by " + name + ": " + jobIds); + System.out.println(); + } catch (IOException e) { + System.out.println("WARN: Error while writing to external child ids file: " + + System.getProperty(EXTERNAL_CHILD_IDS)); + e.printStackTrace(System.out); } - finally { - os.close(); - } - System.out.println(" Hadoop Job IDs executed by " + name + ": " + jobIds.getProperty(HADOOP_JOBS)); - System.out.println(); - } - catch (Exception e) { - System.out.println("WARN: Error getting Hadoop Job IDs executed by " + name); - e.printStackTrace(System.out); + } else { + System.out.println("No child hadoop job is executed."); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java index 680dbfc..1e9d8af 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java @@ -260,7 +260,7 @@ public class ShellMain extends LauncherMain { try { if (needCaptured) { - File file = new File(System.getProperty(LauncherMapper.ACTION_PREFIX + LauncherMapper.ACTION_DATA_OUTPUT_PROPS)); + File file = new File(System.getProperty(OUTPUT_PROPERTIES)); os = new BufferedWriter(new FileWriter(file)); } while ((line = reader.readLine()) != null) { http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java index 90354f3..87c4cc4 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java @@ -47,14 +47,11 @@ import java.util.regex.Pattern; public class PigMain extends LauncherMain { private static final Set<String> DISALLOWED_PIG_OPTIONS = new HashSet<String>(); - public static final String ACTION_PREFIX = "oozie.action."; - public static final String EXTERNAL_CHILD_IDS = ACTION_PREFIX + "externalChildIDs"; - public static final String EXTERNAL_ACTION_STATS = ACTION_PREFIX + "stats.properties"; - public static final String EXTERNAL_STATS_WRITE = ACTION_PREFIX + "external.stats.write"; public static final int STRING_BUFFER_SIZE = 100; private static final Pattern[] PIG_JOB_IDS_PATTERNS = { - Pattern.compile("HadoopJobId: (job_\\S*)") + Pattern.compile("HadoopJobId: (job_\\S*)"), + Pattern.compile("Submitted application (application[0-9_]*)") }; static { @@ -185,6 +182,7 @@ public class PigMain extends LauncherMain { hadoopProps.setProperty("log4j.appender.B.file", logFile); hadoopProps.setProperty("log4j.appender.B.layout", "org.apache.log4j.PatternLayout"); hadoopProps.setProperty("log4j.appender.B.layout.ConversionPattern", "%d [%t] %-5p %c %x - %m%n"); + hadoopProps.setProperty("log4j.logger.org.apache.hadoop.yarn.client.api.impl.YarnClientImpl", "INFO, B"); String localProps = new File("piglog4j.properties").getAbsolutePath(); OutputStream os1 = new FileOutputStream(localProps); http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java index a5291d9..b84e2b5 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java @@ -248,7 +248,7 @@ public class PigMainWithOldAPI extends LauncherMain { // harvesting and recording Hadoop Job IDs Properties jobIds = getHadoopJobIds(logFile); - File file = new File(System.getProperty(LauncherMapper.ACTION_PREFIX + LauncherMapper.ACTION_DATA_OUTPUT_PROPS)); + File file = new File(System.getProperty(OUTPUT_PROPERTIES)); os = new FileOutputStream(file); jobIds.store(os, ""); os.close(); http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java ---------------------------------------------------------------------- diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java index 38e8e8c..bf2869b 100644 --- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java +++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java @@ -212,6 +212,7 @@ public class SparkMain extends LauncherMain { runSpark(sparkArgs.toArray(new String[sparkArgs.size()])); } finally { + System.out.println("\n<<< Invocation of Spark command completed <<<\n"); writeExternalChildIDs(logFile, SPARK_JOB_IDS_PATTERNS, "Spark"); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java ---------------------------------------------------------------------- diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java index 623fd2e..cb65eb8 100644 --- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java +++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java @@ -37,7 +37,9 @@ public class SqoopMain extends LauncherMain { public static final String SQOOP_SITE_CONF = "sqoop-site.xml"; private static final Pattern[] SQOOP_JOB_IDS_PATTERNS = { - Pattern.compile("Job complete: (job_\\S*)"), Pattern.compile("Job (job_\\S*) completed successfully") + Pattern.compile("Job complete: (job_\\S*)"), + Pattern.compile("Job (job_\\S*) has completed successfully"), + Pattern.compile("Submitted application (application[0-9_]*)") }; private static final String SQOOP_LOG4J_PROPS = "sqoop-log4j.properties"; @@ -135,6 +137,7 @@ public class SqoopMain extends LauncherMain { hadoopProps.setProperty("log4j.appender.jobid.layout.ConversionPattern", "%d [%t] %-5p %c %x - %m%n"); hadoopProps.setProperty("log4j.logger.org.apache.hadoop.mapred", "INFO, jobid, A"); hadoopProps.setProperty("log4j.logger.org.apache.hadoop.mapreduce.Job", "INFO, jobid, A"); + hadoopProps.setProperty("log4j.logger.org.apache.hadoop.yarn.client.api.impl.YarnClientImpl", "INFO, jobid"); String localProps = new File(SQOOP_LOG4J_PROPS).getAbsolutePath(); OutputStream os1 = new FileOutputStream(localProps); @@ -182,13 +185,10 @@ public class SqoopMain extends LauncherMain { } } } - - System.out.println(); - System.out.println("<<< Invocation of Sqoop command completed <<<"); - System.out.println(); - - // harvesting and recording Hadoop Job IDs - writeExternalChildIDs(logFile, SQOOP_JOB_IDS_PATTERNS, "Sqoop"); + finally { + System.out.println("\n<<< Invocation of Sqoop command completed <<<\n"); + writeExternalChildIDs(logFile, SQOOP_JOB_IDS_PATTERNS, "Sqoop"); + } } protected void runSqoopJob(String[] args) throws Exception { http://git-wip-us.apache.org/repos/asf/oozie/blob/e8a9b24b/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java b/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java index 6474092..166d939 100644 --- a/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java +++ b/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java @@ -195,7 +195,6 @@ public class TestSqoopActionExecutor extends ActionExecutorTestCase { ae.check(context, context.getAction()); assertTrue(launcherId.equals(context.getAction().getExternalId())); assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); - assertNotNull(context.getAction().getData()); assertNotNull(context.getAction().getExternalChildIDs()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); @@ -215,12 +214,6 @@ public class TestSqoopActionExecutor extends ActionExecutorTestCase { } br.close(); assertEquals(3, count); - - assertNotNull(context.getAction().getData()); - Properties outputData = new Properties(); - outputData.load(new StringReader(context.getAction().getData())); - assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS)); - assertTrue(outputData.getProperty(LauncherMain.HADOOP_JOBS).trim().length() > 0); } public void testSqoopEval() throws Exception { @@ -243,20 +236,13 @@ public class TestSqoopActionExecutor extends ActionExecutorTestCase { ae.check(context, context.getAction()); assertTrue(launcherId.equals(context.getAction().getExternalId())); assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); - assertNotNull(context.getAction().getExternalChildIDs()); - assertEquals(0, context.getAction().getExternalChildIDs().length()); + assertNull(context.getAction().getExternalChildIDs()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); String hadoopCounters = context.getVar(MapReduceActionExecutor.HADOOP_COUNTERS); assertNotNull(hadoopCounters); assertTrue(hadoopCounters.isEmpty()); - - assertNotNull(context.getAction().getData()); - Properties outputData = new Properties(); - outputData.load(new StringReader(context.getAction().getData())); - assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS)); - assertEquals(0, outputData.getProperty(LauncherMain.HADOOP_JOBS).trim().length()); } public void testSqoopActionFreeFormQuery() throws Exception { @@ -306,12 +292,6 @@ public class TestSqoopActionExecutor extends ActionExecutorTestCase { br.close(); } assertEquals(3, count); - - assertNotNull(context.getAction().getData()); - Properties outputData = new Properties(); - outputData.load(new StringReader(context.getAction().getData())); - assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS)); - assertTrue(outputData.getProperty(LauncherMain.HADOOP_JOBS).trim().length() > 0); }