Repository: oozie Updated Branches: refs/heads/master ae2c3009a -> a96e2d1c8
OOZIE-2481 Add YARN_CONF_DIR in the Shell action (harsh) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a96e2d1c Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a96e2d1c Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a96e2d1c Branch: refs/heads/master Commit: a96e2d1c856af5cbf09f0702201ae3aa60050a97 Parents: ae2c300 Author: Harsh J <[email protected]> Authored: Wed Mar 30 19:52:50 2016 +0530 Committer: Harsh J <[email protected]> Committed: Wed Mar 30 19:52:50 2016 +0530 ---------------------------------------------------------------------- .../oozie/action/hadoop/TestShellActionExecutor.java | 10 ++++++++-- docs/src/site/twiki/DG_ShellActionExtension.twiki | 4 ++++ release-log.txt | 1 + .../java/org/apache/oozie/action/hadoop/ShellMain.java | 9 ++++++--- 4 files changed, 19 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/a96e2d1c/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java index ca1dee8..6a962a1 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java @@ -60,8 +60,11 @@ public class TestShellActionExecutor extends ActionExecutorTestCase { : "ls -ltr\necho $1 $2\nexit 1"; private static final String PERL_SCRIPT_CONTENT = "print \"MY_VAR=TESTING\";"; private static final String SHELL_SCRIPT_HADOOP_CONF_DIR_CONTENT = Shell.WINDOWS - ? "echo OOZIE_ACTION_CONF_XML=%OOZIE_ACTION_CONF_XML%\necho HADOOP_CONF_DIR=%HADOOP_CONF_DIR%" - : "echo OOZIE_ACTION_CONF_XML=$OOZIE_ACTION_CONF_XML\necho HADOOP_CONF_DIR=$HADOOP_CONF_DIR"; + ? "echo OOZIE_ACTION_CONF_XML=%OOZIE_ACTION_CONF_XML%\necho HADOOP_CONF_DIR=%HADOOP_CONF_DIR%\n" + : "echo OOZIE_ACTION_CONF_XML=$OOZIE_ACTION_CONF_XML\necho HADOOP_CONF_DIR=$HADOOP_CONF_DIR\n"; + private static final String SHELL_SCRIPT_YARN_CONF_DIR_CONTENT = Shell.WINDOWS + ? "echo OOZIE_ACTION_CONF_XML=%OOZIE_ACTION_CONF_XML%\necho YARN_CONF_DIR=%YARN_CONF_DIR%\n" + : "echo OOZIE_ACTION_CONF_XML=$OOZIE_ACTION_CONF_XML\necho YARN_CONF_DIR=$YARN_CONF_DIR\n"; /** * Verify if the ShellActionExecutor indeed setups the basic stuffs @@ -128,6 +131,7 @@ public class TestShellActionExecutor extends ActionExecutorTestCase { Path script = new Path(getAppPath(), SHELL_SCRIPTNAME); Writer w = new OutputStreamWriter(fs.create(script)); w.write(SHELL_SCRIPT_HADOOP_CONF_DIR_CONTENT); + w.write(SHELL_SCRIPT_YARN_CONF_DIR_CONTENT); w.close(); // Create sample Shell action xml @@ -141,10 +145,12 @@ public class TestShellActionExecutor extends ActionExecutorTestCase { WorkflowAction action = _testSubmit(actionXml, true, ""); String oozieActionConfXml = PropertiesUtils.stringToProperties(action.getData()).getProperty("OOZIE_ACTION_CONF_XML"); String hadoopConfDir = PropertiesUtils.stringToProperties(action.getData()).getProperty("HADOOP_CONF_DIR"); + String yarnConfDir = PropertiesUtils.stringToProperties(action.getData()).getProperty("YARN_CONF_DIR"); assertNotNull(oozieActionConfXml); assertNotNull(hadoopConfDir); String s = new File(oozieActionConfXml).getParent() + File.separator + "oozie-hadoop-conf-"; Assert.assertTrue("Expected HADOOP_CONF_DIR to start with " + s + " but was " + hadoopConfDir, hadoopConfDir.startsWith(s)); + Assert.assertTrue("Expected YARN_CONF_DIR to start with " + s + " but was " + yarnConfDir, yarnConfDir.startsWith(s)); } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/a96e2d1c/docs/src/site/twiki/DG_ShellActionExtension.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_ShellActionExtension.twiki b/docs/src/site/twiki/DG_ShellActionExtension.twiki index 4d8a6b1..025a94d 100644 --- a/docs/src/site/twiki/DG_ShellActionExtension.twiki +++ b/docs/src/site/twiki/DG_ShellActionExtension.twiki @@ -219,6 +219,10 @@ nameNode=hdfs://localhost:8020 queueName=default </verbatim> +---+++ Shell Action Configuration + +=oozie.action.shell.setup.hadoop.conf.dir= - Generates a config directory with various core/hdfs/yarn/mapred-site.xml files and points =HADOOP_CONF_DIR= and =YARN_CONF_DIR= env-vars to it, before the Script is invoked. XML is sourced from the action configuration. Useful when the Shell script passed uses various =hadoop= commands. Default is false. + ---+++ Shell Action Logging Shell action's stdout and stderr output are redirected to the Oozie Launcher map-reduce job task STDOUT that runs the shell command. http://git-wip-us.apache.org/repos/asf/oozie/blob/a96e2d1c/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index b7402be..3de30b3 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2481 Add YARN_CONF_DIR in the Shell action (harsh) OOZIE-2492 JSON security issue in js code (fdenes via rkanter) OOZIE-2429 TestEventGeneration test is flakey (fdenes via rkanter) OOZIE-2466 Repeated failure of TestMetricsInstrumentation.testSamplers (fdenes via rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/a96e2d1c/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java index 553cf1b..e5d8eef 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java @@ -41,6 +41,7 @@ public class ShellMain extends LauncherMain { public static final String CONF_OOZIE_SHELL_SETUP_HADOOP_CONF_DIR = "oozie.action.shell.setup.hadoop.conf.dir"; public static final String OOZIE_ACTION_CONF_XML = "OOZIE_ACTION_CONF_XML"; private static final String HADOOP_CONF_DIR = "HADOOP_CONF_DIR"; + private static final String YARN_CONF_DIR = "YARN_CONF_DIR"; private static String[] HADOOP_SITE_FILES = new String[] {"core-site.xml", "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml"}; @@ -117,9 +118,9 @@ public class ShellMain extends LauncherMain { /** * This method takes the OOZIE_ACTION_CONF_XML and copies it to Hadoop *-site files in a new directory; it then sets the - * HADOOP_CONF_DIR to point there. This should allow most Hadoop ecosystem CLI programs to have the proper configuration, + * HADOOP/YARN_CONF_DIR to point there. This should allow most Hadoop ecosystem CLI programs to have the proper configuration, * propagated from Oozie's copy and including anything set in the Workflow's configuration section as well. Otherwise, - * HADOOP_CONF_DIR points to the NodeManager's *-site files, which are likely not suitable for client programs. + * HADOOP/YARN_CONF_DIR points to the NodeManager's *-site files, which are likely not suitable for client programs. * It will only do this if {@link CONF_OOZIE_SHELL_SETUP_HADOOP_CONF_DIR} is set to true. * * @param actionConf The action configuration @@ -140,8 +141,10 @@ public class ShellMain extends LauncherMain { dstFiles[i] = new File(confDir, HADOOP_SITE_FILES[i]); } copyFileMultiplex(actionXmlFile, dstFiles); - System.out.println("Setting " + HADOOP_CONF_DIR + " to " + confDir.getAbsolutePath()); + System.out.println("Setting " + HADOOP_CONF_DIR + " and " + YARN_CONF_DIR + + " to " + confDir.getAbsolutePath()); envp.put(HADOOP_CONF_DIR, confDir.getAbsolutePath()); + envp.put(YARN_CONF_DIR, confDir.getAbsolutePath()); } } }
