Repository: oozie Updated Branches: refs/heads/master d929be9f7 -> e589521a5
OOZIE-2110 cancel delegation token of launcher jobs that stay till child jobs finish (ryota) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/e589521a Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/e589521a Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/e589521a Branch: refs/heads/master Commit: e589521a56cf758ae26c6b73010f388e47ad61f1 Parents: d929be9 Author: egashira <[email protected]> Authored: Mon Feb 2 10:08:14 2015 -0800 Committer: egashira <[email protected]> Committed: Mon Feb 2 10:08:14 2015 -0800 ---------------------------------------------------------------------- .../org/apache/oozie/action/hadoop/JavaActionExecutor.java | 9 ++++++--- .../apache/oozie/action/hadoop/MapReduceActionExecutor.java | 5 ++++- .../org/apache/oozie/action/hadoop/ShellActionExecutor.java | 1 - .../apache/oozie/action/hadoop/TestJavaActionExecutor.java | 4 ++-- release-log.txt | 1 + .../oozie/action/hadoop/TestMapReduceActionExecutor.java | 5 ++++- 6 files changed, 17 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/e589521a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 6cf6ea3..da550a7 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -478,6 +478,9 @@ public class JavaActionExecutor extends ActionExecutor { setJavaMain(actionConf, actionXml); parseJobXmlAndConfiguration(context, actionXml, appPath, actionConf); + + // set cancel.delegation.token in actionConf that child job doesn't cancel delegation token + actionConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false); return actionConf; } catch (IOException ex) { @@ -821,6 +824,9 @@ public class JavaActionExecutor extends ActionExecutor { // launcher job configuration JobConf launcherJobConf = createBaseHadoopConf(context, actionXml); + // cancel delegation token on a launcher job which stays alive till child job(s) finishes + // otherwise (in mapred action), doesn't cancel not to disturb running child job + launcherJobConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true); setupLauncherConf(launcherJobConf, actionXml, appPathRoot, context); // Properties for when a launcher job's AM gets restarted @@ -898,9 +904,6 @@ public class JavaActionExecutor extends ActionExecutor { // maybe we should add queue to the WF schema, below job-tracker actionConfToLauncherConf(actionConf, launcherJobConf); - // to disable cancelation of delegation token on launcher job end - launcherJobConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false); - return launcherJobConf; } catch (Exception ex) { http://git-wip-us.apache.org/repos/asf/oozie/blob/e589521a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java index 65a4ed2..004adf9 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java @@ -87,7 +87,7 @@ public class MapReduceActionExecutor extends JavaActionExecutor { @Override Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context) throws ActionExecutorException { super.setupLauncherConf(conf, actionXml, appPath, context); - conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true); + conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false); return conf; } @@ -167,6 +167,9 @@ public class MapReduceActionExecutor extends JavaActionExecutor { } } + // child job cancel delegation token for mapred action + actionConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true); + return actionConf; } http://git-wip-us.apache.org/repos/asf/oozie/blob/e589521a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java index d1efda1..3be1eaa 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java @@ -117,7 +117,6 @@ public class ShellActionExecutor extends JavaActionExecutor { protected Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context) throws ActionExecutorException { super.setupLauncherConf(conf, actionXml, appPath, context); - conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true); addDefaultChildEnv(conf); return conf; } http://git-wip-us.apache.org/repos/asf/oozie/blob/e589521a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index c993132..c18d020 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -288,8 +288,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { conf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, actionConf); assertEquals("LQ", conf.get("mapred.job.queue.name")); assertEquals("AQ", actionConf.get("mapred.job.queue.name")); - - + assertEquals(true, conf.getBoolean("mapreduce.job.complete.cancel.delegation.tokens", false)); + assertEquals(false, actionConf.getBoolean("mapreduce.job.complete.cancel.delegation.tokens", true)); } protected Context createContext(String actionXml, String group) throws Exception { http://git-wip-us.apache.org/repos/asf/oozie/blob/e589521a/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 3a049a5..c4a2984 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.2.0 release (trunk - unreleased) +OOZIE-2110 cancel delegation token of launcher jobs that stay till child jobs finish (ryota) OOZIE-2119 Distcp action fails when -D option in arguments (ryota) OOZIE-2112 Child Job URL doesn't show properly with Hive on Tez (ryota) OOZIE-2122 fix test case failure of TestLiteWorkflowAppService (ryota) http://git-wip-us.apache.org/repos/asf/oozie/blob/e589521a/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java b/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java index 50927ce..3204c00 100644 --- a/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java +++ b/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java @@ -229,6 +229,9 @@ public class TestMapReduceActionExecutor extends ActionExecutorTestCase { Configuration conf = ae.createBaseHadoopConf(context, actionXml); ae.setupActionConf(conf, context, actionXml, getFsTestCaseDir()); assertEquals("IN", conf.get("mapred.input.dir")); + JobConf launcherJobConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, conf); + assertEquals(false, launcherJobConf.getBoolean("mapreduce.job.complete.cancel.delegation.tokens", true)); + assertEquals(true, conf.getBoolean("mapreduce.job.complete.cancel.delegation.tokens", false)); // Enable uber jars to test that MapReduceActionExecutor picks up the oozie.mapreduce.uber.jar property correctly Services serv = Services.get(); @@ -239,7 +242,7 @@ public class TestMapReduceActionExecutor extends ActionExecutorTestCase { conf = ae.createBaseHadoopConf(context, actionXml); ae.setupActionConf(conf, context, actionXml, getFsTestCaseDir()); assertEquals(getNameNodeUri() + "/app/job.jar", conf.get("oozie.mapreduce.uber.jar")); // absolute path with namenode - JobConf launcherJobConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, conf); + launcherJobConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, conf); assertEquals(getNameNodeUri() + "/app/job.jar", launcherJobConf.getJar()); // same for launcher conf actionXml = createUberJarActionXML("/app/job.jar", "");
