OOZIE-2205 add option to load default/site.xml to actionConf on compute node (ryota)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/c94ff921 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/c94ff921 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/c94ff921 Branch: refs/heads/master Commit: c94ff9210998c608d05fd4de35b432a25ea7f185 Parents: 8707cad Author: egashira <ryota.egash...@yahoo.com> Authored: Sat Apr 18 00:13:24 2015 -0700 Committer: egashira <ryota.egash...@yahoo.com> Committed: Sat Apr 18 00:13:24 2015 -0700 ---------------------------------------------------------------------- .../oozie/action/hadoop/HiveActionExecutor.java | 11 +++++++++ .../oozie/action/hadoop/JavaActionExecutor.java | 21 +++++++++++++--- .../oozie/action/hadoop/PigActionExecutor.java | 13 ++++++++++ .../oozie/service/HadoopAccessorService.java | 1 + core/src/main/resources/oozie-default.xml | 13 ++++++++++ release-log.txt | 1 + .../action/hadoop/TestHiveActionExecutor.java | 26 ++++++++++++++++++++ .../action/hadoop/TestPigActionExecutor.java | 15 +++++++++++ 8 files changed, 97 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c94ff921/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java index dbb50ba..bc5b033 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java @@ -30,11 +30,15 @@ import java.util.Properties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.action.ActionExecutorException; +import org.apache.oozie.action.ActionExecutor.Context; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.client.XOozieClient; +import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.HadoopAccessorException; +import org.apache.oozie.service.HadoopAccessorService; import org.jdom.Element; import org.jdom.JDOMException; import org.jdom.Namespace; @@ -138,4 +142,11 @@ public class HiveActionExecutor extends ScriptLanguageActionExecutor { return new String[]{"hive-site.xml"}; } + @Override + protected JobConf loadHadoopDefaultResources(Context context, Element actionXml) { + boolean loadDefaultResources = ConfigurationService + .getBoolean(HadoopAccessorService.ACTION_CONFS_LOAD_DEFAULT_RESOURCES); + JobConf conf = super.createBaseHadoopConf(context, actionXml, loadDefaultResources); + return conf; + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/c94ff921/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 163956e..eb2dbdb 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -82,7 +82,7 @@ import org.apache.oozie.hadoop.utils.HadoopShims; public class JavaActionExecutor extends ActionExecutor { - private static final String HADOOP_USER = "user.name"; + protected static final String HADOOP_USER = "user.name"; public static final String HADOOP_JOB_TRACKER = "mapred.job.tracker"; public static final String HADOOP_JOB_TRACKER_2 = "mapreduce.jobtracker.address"; public static final String HADOOP_YARN_RM = "yarn.resourcemanager.address"; @@ -203,10 +203,20 @@ public class JavaActionExecutor extends ActionExecutor { } public JobConf createBaseHadoopConf(Context context, Element actionXml) { + return createBaseHadoopConf(context, actionXml, true); + } + + protected JobConf createBaseHadoopConf(Context context, Element actionXml, boolean loadResources) { Namespace ns = actionXml.getNamespace(); String jobTracker = actionXml.getChild("job-tracker", ns).getTextTrim(); String nameNode = actionXml.getChild("name-node", ns).getTextTrim(); - JobConf conf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker); + JobConf conf = null; + if (loadResources) { + conf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker); + } + else { + conf = new JobConf(false); + } conf.set(HADOOP_USER, context.getProtoActionConf().get(WorkflowAppService.HADOOP_USER)); conf.set(HADOOP_JOB_TRACKER, jobTracker); conf.set(HADOOP_JOB_TRACKER_2, jobTracker); @@ -216,6 +226,10 @@ public class JavaActionExecutor extends ActionExecutor { return conf; } + protected JobConf loadHadoopDefaultResources(Context context, Element actionXml) { + return createBaseHadoopConf(context, actionXml); + } + private void injectLauncherProperties(Configuration srcConf, Configuration launcherConf) { for (Map.Entry<String, String> entry : srcConf) { if (entry.getKey().startsWith("oozie.launcher.")) { @@ -512,7 +526,6 @@ public class JavaActionExecutor extends ActionExecutor { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); XConfiguration actionDefaults = has.createActionDefaultConf(actionConf.get(HADOOP_JOB_TRACKER), getType()); XConfiguration.injectDefaults(actionDefaults, actionConf); - has.checkSupportedFilesystem(appPath.toUri()); // Set the Java Main Class for the Java action to give to the Java launcher @@ -1009,7 +1022,7 @@ public class JavaActionExecutor extends ActionExecutor { Element actionXml = XmlUtils.parseXml(action.getConf()); // action job configuration - Configuration actionConf = createBaseHadoopConf(context, actionXml); + Configuration actionConf = loadHadoopDefaultResources(context, actionXml); setupActionConf(actionConf, context, actionXml, appPathRoot); LOG.debug("Setting LibFilesArchives "); setLibFilesArchives(context, actionXml, appPathRoot, actionConf); http://git-wip-us.apache.org/repos/asf/oozie/blob/c94ff921/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java index 4d6a1b4..8b2dc16 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java @@ -20,9 +20,15 @@ package org.apache.oozie.action.hadoop; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.JobConf; import org.apache.oozie.action.ActionExecutorException; +import org.apache.oozie.action.ActionExecutor.Context; import org.apache.oozie.client.XOozieClient; import org.apache.oozie.client.WorkflowAction; +import org.apache.oozie.service.ConfigurationService; +import org.apache.oozie.service.HadoopAccessorService; +import org.apache.oozie.service.Services; +import org.apache.oozie.service.WorkflowAppService; import org.jdom.Element; import org.jdom.Namespace; import org.jdom.JDOMException; @@ -120,4 +126,11 @@ public class PigActionExecutor extends ScriptLanguageActionExecutor { return XOozieClient.PIG_SCRIPT; } + @Override + protected JobConf loadHadoopDefaultResources(Context context, Element actionXml) { + boolean loadDefaultResources = ConfigurationService + .getBoolean(HadoopAccessorService.ACTION_CONFS_LOAD_DEFAULT_RESOURCES); + JobConf conf = super.createBaseHadoopConf(context, actionXml, loadDefaultResources); + return conf; + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/c94ff921/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java index 102699b..6a3f045 100644 --- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java +++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java @@ -68,6 +68,7 @@ public class HadoopAccessorService implements Service { public static final String NAME_NODE_WHITELIST = CONF_PREFIX + "nameNode.whitelist"; public static final String HADOOP_CONFS = CONF_PREFIX + "hadoop.configurations"; public static final String ACTION_CONFS = CONF_PREFIX + "action.configurations"; + public static final String ACTION_CONFS_LOAD_DEFAULT_RESOURCES = ACTION_CONFS + ".load.default.resources"; public static final String KERBEROS_AUTH_ENABLED = CONF_PREFIX + "kerberos.enabled"; public static final String KERBEROS_KEYTAB = CONF_PREFIX + "keytab.file"; public static final String KERBEROS_PRINCIPAL = CONF_PREFIX + "kerberos.principal"; http://git-wip-us.apache.org/repos/asf/oozie/blob/c94ff921/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index ffff7c3..ecc0d47 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -1877,6 +1877,19 @@ </description> </property> + <property> + <name>oozie.service.HadoopAccessorService.action.configurations.load.default.resources</name> + <value>true</value> + <description> + true means that default and site xml files of hadoop (core-default, core-site, + hdfs-default, hdfs-site, mapred-default, mapred-site, yarn-default, yarn-site) + are parsed into actionConf on Oozie server. false means that site xml files are + not loaded on server, instead loaded on launcher node. + This is only done for pig and hive actions which handle loading those files + automatically from the classpath on launcher task. It defaults to true. + </description> + </property> + <!-- Credentials --> <property> <name>oozie.credentials.credentialclasses</name> http://git-wip-us.apache.org/repos/asf/oozie/blob/c94ff921/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 6fa00ca..8586dc7 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.2.0 release (trunk - unreleased) +OOZIE-2205 add option to load default/site.xml to actionConf on compute node (ryota) OOZIE-2206 Change Reaper mode on ChildReaper in ZKLocksService (ryota) OOZIE-2209 Oozie jobs to set "java.io.tmpdir" to "./tmp" (kailongs via rohini) OOZIE-2051 Allows multiple default action configuration files per action (venkatnrangan via bzhang) http://git-wip-us.apache.org/repos/asf/oozie/blob/c94ff921/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java index c28839c..a11d409 100644 --- a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java +++ b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Properties; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobClient; @@ -38,7 +39,9 @@ import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.mapred.RunningJob; import org.apache.oozie.WorkflowActionBean; import org.apache.oozie.WorkflowJobBean; +import org.apache.oozie.action.hadoop.ActionExecutorTestCase.Context; import org.apache.oozie.client.WorkflowAction; +import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.HadoopAccessorService; import org.apache.oozie.service.Services; import org.apache.oozie.service.WorkflowAppService; @@ -227,4 +230,27 @@ public class TestHiveActionExecutor extends ActionExecutorTestCase { return new Context(wf, action); } + public void testActionConfLoadDefaultResources() throws Exception { + ConfigurationService.setBoolean( + "oozie.service.HadoopAccessorService.action.configurations.load.default.resources", false); + Path inputDir = new Path(getFsTestCaseDir(), INPUT_DIRNAME); + Path outputDir = new Path(getFsTestCaseDir(), OUTPUT_DIRNAME); + + FileSystem fs = getFileSystem(); + Path script = new Path(getAppPath(), HIVE_SCRIPT_FILENAME); + Writer scriptWriter = new OutputStreamWriter(fs.create(script)); + scriptWriter.write(getHiveScript(inputDir.toString(), outputDir.toString())); + scriptWriter.close(); + + Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME))); + dataWriter.write(SAMPLE_DATA_TEXT); + dataWriter.close(); + + Context context = createContext(getActionXml()); + submitAction(context); + FSDataInputStream os = fs.open(new Path(context.getActionDir(), LauncherMapper.ACTION_CONF_XML)); + XConfiguration conf = new XConfiguration(); + conf.addResource(os); + assertNull(conf.get("oozie.HadoopAccessorService.created")); + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/c94ff921/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java index 039bd84..4b9a4e8 100644 --- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java +++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java @@ -20,6 +20,7 @@ package org.apache.oozie.action.hadoop; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.mapred.JobConf; @@ -29,6 +30,7 @@ import org.apache.hadoop.mapred.JobID; import org.apache.oozie.WorkflowActionBean; import org.apache.oozie.WorkflowJobBean; import org.apache.oozie.client.WorkflowAction; +import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.URIHandlerService; import org.apache.oozie.service.WorkflowAppService; import org.apache.oozie.service.Services; @@ -247,6 +249,19 @@ public class TestPigActionExecutor extends ActionExecutorTestCase { } + public void testActionConfLoadDefaultResources() throws Exception { + ConfigurationService.setBoolean( + "oozie.service.HadoopAccessorService.action.configurations.load.default.resources", false); + String actionXml = setPigActionXml(PIG_SCRIPT, true); + Context context = createContext(actionXml); + submitAction(context); + FileSystem fs = getFileSystem(); + FSDataInputStream os = fs.open(new Path(context.getActionDir(), LauncherMapper.ACTION_CONF_XML)); + XConfiguration conf = new XConfiguration(); + conf.addResource(os); + assertNull(conf.get("oozie.HadoopAccessorService.created")); + } + /* * Test the Hadoop IDs obtained from the Pig job */
