Repository: oozie Updated Branches: refs/heads/master d5b13db2b -> 47d62743a
OOZIE-1740 Add a new function hadoop:conf() that can be invoked from the workflow.xml and will return a hadoop configuration option (sam liu via rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/47d62743 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/47d62743 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/47d62743 Branch: refs/heads/master Commit: 47d62743a8b639f65226c5e78455989605f5aabf Parents: d5b13db Author: Robert Kanter <[email protected]> Authored: Mon Jul 14 14:30:27 2014 -0700 Committer: Robert Kanter <[email protected]> Committed: Mon Jul 14 14:30:27 2014 -0700 ---------------------------------------------------------------------- .../oozie/action/hadoop/HadoopELFunctions.java | 16 +++++++++ core/src/main/resources/oozie-default.xml | 31 ++++++++++++------ .../action/hadoop/TestHadoopELFunctions.java | 34 ++++++++++++++++++++ .../src/site/twiki/WorkflowFunctionalSpec.twiki | 30 +++++++++++++++++ release-log.txt | 1 + 5 files changed, 102 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/47d62743/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java b/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java index d325535..83813a1 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java @@ -17,7 +17,10 @@ */ package org.apache.oozie.action.hadoop; +import org.apache.hadoop.conf.Configuration; import org.apache.oozie.DagELFunctions; +import org.apache.oozie.service.HadoopAccessorService; +import org.apache.oozie.service.Services; import org.apache.oozie.util.ELEvaluationException; import org.apache.oozie.util.XLog; import org.apache.oozie.workflow.WorkflowInstance; @@ -58,6 +61,19 @@ public class HadoopELFunctions { return counters; } + public static String hadoop_conf(String hadoopConfHostPort, String propName) { + Configuration conf = Services.get().get(HadoopAccessorService.class) + .createJobConf(hadoopConfHostPort); + String prop = conf.get(propName); + if (prop == null || prop.equals("")) { + conf = new Configuration(); + prop = conf.get(propName); + } + if (prop == null) + prop = ""; + return prop; + } + @SuppressWarnings("unchecked") private static Map<String, Map<String, Long>> getCounters(String nodeName) throws ELEvaluationException { String jsonCounters = DagELFunctions.getActionVar(nodeName, MapReduceActionExecutor.HADOOP_COUNTERS); http://git-wip-us.apache.org/repos/asf/oozie/blob/47d62743/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index b38801f..2b68436 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -616,6 +616,7 @@ wf:actionTrackerUri=org.apache.oozie.DagELFunctions#wf_actionTrackerUri, wf:actionExternalStatus=org.apache.oozie.DagELFunctions#wf_actionExternalStatus, hadoop:counters=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_counters, + hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf, fs:exists=org.apache.oozie.action.hadoop.FsELFunctions#fs_exists, fs:isDir=org.apache.oozie.action.hadoop.FsELFunctions#fs_isDir, fs:dirSize=org.apache.oozie.action.hadoop.FsELFunctions#fs_dirSize, @@ -720,7 +721,8 @@ coord:endOfDays=org.apache.oozie.coord.CoordELFunctions#ph1_coord_endOfDays, coord:endOfMonths=org.apache.oozie.coord.CoordELFunctions#ph1_coord_endOfMonths, coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, - coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user + coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user, + hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf </value> <description> EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. @@ -733,7 +735,8 @@ coord:days=org.apache.oozie.coord.CoordELFunctions#ph1_coord_days, coord:months=org.apache.oozie.coord.CoordELFunctions#ph1_coord_months, coord:hours=org.apache.oozie.coord.CoordELFunctions#ph1_coord_hours, - coord:minutes=org.apache.oozie.coord.CoordELFunctions#ph1_coord_minutes + coord:minutes=org.apache.oozie.coord.CoordELFunctions#ph1_coord_minutes, + hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf </value> <description> EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. @@ -780,7 +783,8 @@ <name>oozie.service.ELService.functions.coord-job-submit-nofuncs</name> <value> coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, - coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user + coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user, + hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf </value> <description> EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. @@ -833,7 +837,8 @@ coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo, coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user, - coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph1_coord_absolute_echo + coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph1_coord_absolute_echo, + hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf </value> <description> EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. @@ -894,7 +899,8 @@ coord:dataInPartitionMax=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataInPartitionMax_echo, coord:dataInPartitions=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataInPartitions_echo, coord:dataOutPartitions=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataOutPartitions_echo, - coord:dataOutPartitionValue=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataOutPartitionValue_echo + coord:dataOutPartitionValue=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataOutPartitionValue_echo, + hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf </value> <description> EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. @@ -950,7 +956,8 @@ coord:databaseOut=org.apache.oozie.coord.HCatELFunctions#ph1_coord_databaseOut_echo, coord:tableOut=org.apache.oozie.coord.HCatELFunctions#ph1_coord_tableOut_echo, coord:dataOutPartitions=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataOutPartitions_echo, - coord:dataOutPartitionValue=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataOutPartitionValue_echo + coord:dataOutPartitionValue=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataOutPartitionValue_echo, + hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf </value> <description> EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. @@ -1006,7 +1013,8 @@ coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user, coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_echo, - coord:absoluteRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_range + coord:absoluteRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_range, + hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf </value> <description> EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. @@ -1062,7 +1070,8 @@ coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user, coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_echo, - coord:absoluteRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_range + coord:absoluteRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_range, + hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf </value> <description> EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. @@ -1117,7 +1126,8 @@ coord:databaseOut=org.apache.oozie.coord.HCatELFunctions#ph3_coord_databaseOut, coord:tableOut=org.apache.oozie.coord.HCatELFunctions#ph3_coord_tableOut, coord:dataOutPartitions=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataOutPartitions, - coord:dataOutPartitionValue=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataOutPartitionValue + coord:dataOutPartitionValue=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataOutPartitionValue, + hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf </value> <description> EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. @@ -1183,7 +1193,8 @@ coord:dataInPartitionMax=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataInPartitionMax, coord:dataInPartitions=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataInPartitions, coord:dataOutPartitions=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataOutPartitions, - coord:dataOutPartitionValue=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataOutPartitionValue + coord:dataOutPartitionValue=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataOutPartitionValue, + hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf </value> <description> EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. http://git-wip-us.apache.org/repos/asf/oozie/blob/47d62743/core/src/test/java/org/apache/oozie/action/hadoop/TestHadoopELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestHadoopELFunctions.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestHadoopELFunctions.java index 8212f09..3bb35a5 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestHadoopELFunctions.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestHadoopELFunctions.java @@ -33,6 +33,7 @@ import org.apache.oozie.workflow.lite.EndNodeDef; import org.apache.oozie.workflow.lite.LiteWorkflowApp; import org.apache.oozie.workflow.lite.LiteWorkflowInstance; import org.apache.oozie.workflow.lite.StartNodeDef; +import java.io.ByteArrayOutputStream; import java.util.HashMap; public class TestHadoopELFunctions extends ActionExecutorTestCase { @@ -163,4 +164,37 @@ public class TestHadoopELFunctions extends ActionExecutorTestCase { assertEquals(new Long(33), eval.evaluate("${hadoop:counters('H')['job_201111300933_0004']['MAP_INPUT_RECORDS']}", Long.class)); } + + public void testHadoopConfFunctions() throws Exception { + XConfiguration jobConf = new XConfiguration(); + XConfiguration.copy(createJobConf(), jobConf); + + String testHadoopOptionValue = jobConf.get("mapred.tasktracker.map.tasks.maximum"); + jobConf.set("test.name.node.uri", getNameNodeUri()); + jobConf.set("test.hadoop.option", "mapred.tasktracker.map.tasks.maximum"); + + WorkflowJobBean workflow = new WorkflowJobBean(); + workflow.setProtoActionConf("<configuration/>"); + LiteWorkflowApp wfApp = new LiteWorkflowApp("x", "<workflow-app/>", + new StartNodeDef( + LiteWorkflowStoreService.LiteControlNodeHandler.class, "a")); + wfApp.addNode(new EndNodeDef("a", + LiteWorkflowStoreService.LiteControlNodeHandler.class)); + WorkflowInstance wi = new LiteWorkflowInstance(wfApp, jobConf, "1"); + + workflow.setWorkflowInstance(wi); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + jobConf.writeXml(baos); + workflow.setProtoActionConf(baos.toString()); + + final WorkflowActionBean action = new WorkflowActionBean(); + ELEvaluator eval = Services.get().get(ELService.class).createEvaluator( + "workflow"); + DagELFunctions.configureEvaluator(eval, workflow, action); + + assertEquals(testHadoopOptionValue, + eval.evaluate("${hadoop:conf(wf:conf('test.name.node.uri'), wf:conf('test.hadoop.option'))}", + String.class)); + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/47d62743/docs/src/site/twiki/WorkflowFunctionalSpec.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki index f0eb393..3319bcc 100644 --- a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki +++ b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki @@ -2011,6 +2011,36 @@ Below is the workflow that describes how to access specific information using ha </workflow-app> </verbatim> +#Hadoop Configuration EL Function +*String hadoop:conf(String hadoopConfHostPort, String propName)* + +It returns the value of a property of Hadoop configuration. + +The hadoopConfHostPort is the 'host:port' of a Hadoop cluster, such as 'NameNodeHostAddress:Port' and +'JobTrackerHostAddress:Port'. The propName is the name of target property. If hadoopConfHostPort could +be connected, Hadoop Conf EL Function will return the value of propName from the specific host address +and port. If hadoopConfHostPort could not be connected, Hadoop Conf EL Function will generate a default +hadoop configuration object directly, and then return the value of target property. + +*Example of usage of Hadoop Configuration EL Function:* + +<verbatim> +<action name="mr-node"> + <map-reduce> + <job-tracker>${jobTracker}</job-tracker> + <name-node>${nameNode}</name-node> + <configuration> + <property> + <name>mapreduce.map.java.opts</name> + <value>${hadoop:conf("9.181.7.69:9001","mapreduce.map.java.opts")} -Xss512k </value> + </property> + </configuration> + </map-reduce> + <ok to="end"/> + <error to="fail"/> +</action> +</verbatim> + ---++++ 4.2.6 Hadoop Jobs EL Function The function _wf:actionData()_ can be used to access Hadoop ID's for actions such as Pig, by specifying the key as _hadoopJobs_. http://git-wip-us.apache.org/repos/asf/oozie/blob/47d62743/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 81559eb..1f7c322 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.1.0 release (trunk - unreleased) +OOZIE-1740 Add a new function hadoop:conf() that can be invoked from the workflow.xml and will return a hadoop configuration option (sam liu via rkanter) OOZIE-1911 SLA calculation in HA mode does wrong bit comparison for 'start' and 'duration' (mna) OOZIE-1926 make gz blob compression as default (ryota) OOZIE-1916 Use Curator leader latch instead of checking the order of Oozie servers (rkanter)
