OOZIE-2592 fix Hive2&Sqoop action when System.exit() called Change-Id: Ia5db070d5c5287316393d56c626300c4e7777f34
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5c4a79be Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5c4a79be Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5c4a79be Branch: refs/heads/oya Commit: 5c4a79bea4a9ec889c9856c4ef986c846454cc26 Parents: ddbd90f Author: Peter Bacsko <[email protected]> Authored: Tue Nov 22 15:39:00 2016 +0100 Committer: Peter Bacsko <[email protected]> Committed: Tue Nov 22 16:55:39 2016 +0100 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 44 ---------------- .../apache/oozie/action/hadoop/HiveMain.java | 7 --- .../apache/oozie/action/hadoop/Hive2Main.java | 10 ++-- .../action/hadoop/TestHive2ActionExecutor.java | 55 +++++++++++--------- .../apache/oozie/action/hadoop/SqoopMain.java | 7 --- 5 files changed, 32 insertions(+), 91 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/5c4a79be/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 2ec5266..56226ee 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -37,7 +37,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.Set; -import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; @@ -52,8 +51,6 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.JobID; -import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapred.TaskLog; import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager; import org.apache.hadoop.mapreduce.v2.util.MRApps; @@ -81,7 +78,6 @@ import org.apache.oozie.action.ActionExecutor; import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.WorkflowAction; -import org.apache.oozie.client.WorkflowJob; import org.apache.oozie.command.coord.CoordActionStartXCommand; import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.HadoopAccessorException; @@ -333,46 +329,6 @@ public class JavaActionExecutor extends ActionExecutor { } } - // FIXME: is this needed? - private HashMap<String, List<String>> populateEnvMap(String input) { - HashMap<String, List<String>> envMaps = new HashMap<String, List<String>>(); - String[] envEntries = input.split(","); - for (String envEntry : envEntries) { - String[] envKeyVal = envEntry.split("="); - String envKey = envKeyVal[0].trim(); - List<String> valList = envMaps.get(envKey); - if (valList == null) { - valList = new ArrayList<String>(); - } - valList.add(envKeyVal[1].trim()); - envMaps.put(envKey, valList); - } - return envMaps; - } - - public int extractHeapSizeMB(String input) { - int ret = 0; - if(input == null || input.equals("")) - return ret; - Matcher m = heapPattern.matcher(input); - String heapStr = null; - String heapNum = null; - // Grabs the last match which takes effect (in case that multiple Xmx options specified) - while (m.find()) { - heapStr = m.group(1); - heapNum = m.group(2); - } - if (heapStr != null) { - // when Xmx specified in Gigabyte - if(heapStr.endsWith("g") || heapStr.endsWith("G")) { - ret = Integer.parseInt(heapNum) * 1024; - } else { - ret = Integer.parseInt(heapNum); - } - } - return ret; - } - public static void parseJobXmlAndConfiguration(Context context, Element element, Path appPath, Configuration conf) throws IOException, ActionExecutorException, HadoopAccessorException, URISyntaxException { parseJobXmlAndConfiguration(context, element, appPath, conf, false); http://git-wip-us.apache.org/repos/asf/oozie/blob/5c4a79be/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java index f59750b..242cd6c 100644 --- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java +++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java @@ -317,13 +317,6 @@ public class HiveMain extends LauncherMain { try { runHive(arguments.toArray(new String[arguments.size()])); } - catch (SecurityException ex) { - if (LauncherSecurityManager.getExitInvoked()) { - if (LauncherSecurityManager.getExitCode() != 0) { - throw ex; - } - } - } finally { System.out.println("\n<<< Invocation of Hive command completed <<<\n"); writeExternalChildIDs(logFile, HIVE_JOB_IDS_PATTERNS, "Hive"); http://git-wip-us.apache.org/repos/asf/oozie/blob/5c4a79be/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java ---------------------------------------------------------------------- diff --git a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java index ccf2aff..b418b89 100644 --- a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java +++ b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java @@ -240,13 +240,6 @@ public class Hive2Main extends LauncherMain { try { runBeeline(arguments.toArray(new String[arguments.size()]), logFile); } - catch (SecurityException ex) { - if (LauncherSecurityManager.getExitInvoked()) { - if (LauncherSecurityManager.getExitCode() != 0) { - throw ex; - } - } - } finally { System.out.println("\n<<< Invocation of Beeline command completed <<<\n"); writeExternalChildIDs(logFile, HIVE2_JOB_IDS_PATTERNS, "Beeline"); @@ -265,6 +258,9 @@ public class Hive2Main extends LauncherMain { BeeLine beeLine = new BeeLine(); beeLine.setErrorStream(new PrintStream(new TeeOutputStream(System.err, new FileOutputStream(logFile)))); int status = beeLine.begin(args, null); + System.out.println("*** Return value from Beeline is: " + status); + System.out.println("*** Current security manager: " + System.getSecurityManager()); + beeLine.close(); if (status != 0) { System.exit(status); } http://git-wip-us.apache.org/repos/asf/oozie/blob/5c4a79be/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java index 030942a..89bcd45 100644 --- a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java +++ b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java @@ -62,10 +62,9 @@ public class TestHive2ActionExecutor extends ActionExecutorTestCase { setSystemProperty("oozie.service.ActionService.executor.classes", Hive2ActionExecutor.class.getName()); } - @SuppressWarnings("unchecked") public void testSetupMethodsForScript() throws Exception { Hive2ActionExecutor ae = new Hive2ActionExecutor(); - List<Class> classes = new ArrayList<Class>(); + List<Class<?>> classes = new ArrayList<>(); classes.add(Hive2Main.class); assertEquals(classes, ae.getLauncherClasses()); @@ -103,10 +102,9 @@ public class TestHive2ActionExecutor extends ActionExecutorTestCase { assertEquals("--dee", conf.get("oozie.hive2.args.1")); } - @SuppressWarnings("unchecked") public void testSetupMethodsForQuery() throws Exception { Hive2ActionExecutor ae = new Hive2ActionExecutor(); - List<Class> classes = new ArrayList<Class>(); + List<Class<?>> classes = new ArrayList<>(); classes.add(Hive2Main.class); assertEquals(classes, ae.getLauncherClasses()); @@ -245,28 +243,33 @@ public class TestHive2ActionExecutor extends ActionExecutorTestCase { assertTrue(fs.exists(outputDir)); assertTrue(fs.isDirectory(outputDir)); } - // Negative testcase with incorrect hive-query. - { - String query = getHive2BadScript(inputDir.toString(), outputDir.toString()); - Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME))); - dataWriter.write(SAMPLE_DATA_TEXT); - dataWriter.close(); - Context context = createContext(getQueryActionXml(query)); - final String launcherId = submitAction(context, Namespace.getNamespace("uri:oozie:hive2-action:0.2")); - waitUntilYarnAppDoneAndAssertSuccess(launcherId); - Configuration conf = new XConfiguration(); - conf.set("user.name", getTestUser()); - Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), - conf); - assertFalse(LauncherMapperHelper.hasIdSwap(actionData)); - Hive2ActionExecutor ae = new Hive2ActionExecutor(); - ae.check(context, context.getAction()); - assertTrue(launcherId.equals(context.getAction().getExternalId())); - assertEquals("FAILED/KILLED", context.getAction().getExternalStatus()); - ae.end(context, context.getAction()); - assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus()); - assertNull(context.getExternalChildIDs()); - } + } + + public void testHive2ActionFails() throws Exception { + setupHiveServer2(); + Path inputDir = new Path(getFsTestCaseDir(), INPUT_DIRNAME); + Path outputDir = new Path(getFsTestCaseDir(), OUTPUT_DIRNAME); + FileSystem fs = getFileSystem(); + + String query = getHive2BadScript(inputDir.toString(), outputDir.toString()); + Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME))); + dataWriter.write(SAMPLE_DATA_TEXT); + dataWriter.close(); + Context context = createContext(getQueryActionXml(query)); + final String launcherId = submitAction(context, Namespace.getNamespace("uri:oozie:hive2-action:0.2")); + waitUntilYarnAppDoneAndAssertSuccess(launcherId); + Configuration conf = new XConfiguration(); + conf.set("user.name", getTestUser()); + Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), + conf); + assertFalse(LauncherMapperHelper.hasIdSwap(actionData)); + Hive2ActionExecutor ae = new Hive2ActionExecutor(); + ae.check(context, context.getAction()); + assertTrue(launcherId.equals(context.getAction().getExternalId())); + assertEquals("FAILED/KILLED", context.getAction().getExternalStatus()); + ae.end(context, context.getAction()); + assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus()); + assertNull(context.getExternalChildIDs()); } private String getHive2BadScript(String inputPath, String outputPath) { http://git-wip-us.apache.org/repos/asf/oozie/blob/5c4a79be/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java ---------------------------------------------------------------------- diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java index 9c6ec6c..92c960f 100644 --- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java +++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java @@ -182,13 +182,6 @@ public class SqoopMain extends LauncherMain { try { runSqoopJob(sqoopArgs); } - catch (SecurityException ex) { - if (LauncherSecurityManager.getExitInvoked()) { - if (LauncherSecurityManager.getExitCode() != 0) { - throw ex; - } - } - } finally { System.out.println("\n<<< Invocation of Sqoop command completed <<<\n"); writeExternalChildIDs(logFile, SQOOP_JOB_IDS_PATTERNS, "Sqoop");
