Repository: oozie Updated Branches: refs/heads/master 9b1f6b653 -> 97b21af4d
OOZIE-2133 Support getting ATS delegation tokens for tez jobs (rohini) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/97b21af4 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/97b21af4 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/97b21af4 Branch: refs/heads/master Commit: 97b21af4d305bdb12f0642f641c6ad18d3abfa15 Parents: 9b1f6b6 Author: Rohini Palaniswamy <[email protected]> Authored: Mon Feb 9 12:27:42 2015 -0800 Committer: Rohini Palaniswamy <[email protected]> Committed: Mon Feb 9 12:27:42 2015 -0800 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 14 +++ .../oozie/service/ConfigurationService.java | 15 +++ core/src/main/resources/oozie-default.xml | 12 +++ .../action/hadoop/TestJavaActionExecutor.java | 99 +++++++++++++++++--- release-log.txt | 1 + 5 files changed, 129 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/97b21af4/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 3383522..5f8646d 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -93,6 +93,7 @@ public class JavaActionExecutor extends ActionExecutor { public final static String MAX_EXTERNAL_STATS_SIZE = "oozie.external.stats.max.size"; public static final String ACL_VIEW_JOB = "mapreduce.job.acl-view-job"; public static final String ACL_MODIFY_JOB = "mapreduce.job.acl-modify-job"; + public static final String HADOOP_YARN_TIMELINE_SERVICE_ENABLED = "yarn.timeline-service.enabled"; public static final String HADOOP_YARN_UBER_MODE = "mapreduce.job.ubertask.enable"; public static final String HADOOP_MAP_MEMORY_MB = "mapreduce.map.memory.mb"; public static final String HADOOP_CHILD_JAVA_OPTS = "mapred.child.java.opts"; @@ -279,6 +280,17 @@ public class JavaActionExecutor extends ActionExecutor { } } + void injectLauncherTimelineServiceEnabled(Configuration launcherConf, Configuration actionConf) { + // Getting delegation token for ATS. If tez-site.xml is present in distributed cache, turn on timeline service. + if (actionConf.get("oozie.launcher." + HADOOP_YARN_TIMELINE_SERVICE_ENABLED) == null + && ConfigurationService.getBoolean("oozie.action.launcher." + HADOOP_YARN_TIMELINE_SERVICE_ENABLED)) { + String cacheFiles = launcherConf.get("mapred.cache.files"); + if (cacheFiles != null && cacheFiles.contains("tez-site.xml")) { + launcherConf.setBoolean(HADOOP_YARN_TIMELINE_SERVICE_ENABLED, true); + } + } + } + void updateConfForUberMode(Configuration launcherConf) { // child.env @@ -907,6 +919,8 @@ public class JavaActionExecutor extends ActionExecutor { } } + injectLauncherTimelineServiceEnabled(launcherJobConf, actionConf); + // properties from action that are needed by the launcher (e.g. QUEUE NAME, ACLs) // maybe we should add queue to the WF schema, below job-tracker actionConfToLauncherConf(actionConf, launcherJobConf); http://git-wip-us.apache.org/repos/asf/oozie/blob/97b21af4/core/src/main/java/org/apache/oozie/service/ConfigurationService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/ConfigurationService.java b/core/src/main/java/org/apache/oozie/service/ConfigurationService.java index d710744..d602e53 100644 --- a/core/src/main/java/org/apache/oozie/service/ConfigurationService.java +++ b/core/src/main/java/org/apache/oozie/service/ConfigurationService.java @@ -36,8 +36,11 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.Arrays; + import org.apache.oozie.util.ZKUtils; +import com.google.common.annotations.VisibleForTesting; + /** * Built in service that initializes the services configuration. * <p/> @@ -459,6 +462,18 @@ public class ConfigurationService implements Service, Instrumentable { } } + @VisibleForTesting + public static void set(String name, String value) { + Configuration conf = Services.get().getConf(); + conf.set(name, value); + } + + @VisibleForTesting + public static void setBoolean(String name, boolean value) { + Configuration conf = Services.get().getConf(); + conf.setBoolean(name, value); + } + public static String get(String name) { Configuration conf = Services.get().getConf(); return get(conf, name); http://git-wip-us.apache.org/repos/asf/oozie/blob/97b21af4/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 17da11b..207a912 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -1700,6 +1700,18 @@ </description> </property> + <property> + <name>oozie.action.launcher.yarn.timeline-service.enabled</name> + <value>false</value> + <description> + Enables/disables getting delegation tokens for ATS for the launcher job in + YARN/Hadoop 2.6 (no effect in Hadoop 1) for all action types by default if tez-site.xml is present in + distributed cache. + This can be overridden on a per-action basis by setting + oozie.launcher.yarn.timeline-service.enabled in an action's configuration section in a workflow. + </description> + </property> + <!-- HadoopActionExecutor --> <!-- This is common to the subclasses action executors for map-reduce and pig --> http://git-wip-us.apache.org/repos/asf/oozie/blob/97b21af4/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 7ee865b..f4d055f 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -33,6 +33,7 @@ import java.util.Properties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; @@ -49,6 +50,7 @@ import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.client.WorkflowJob; +import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.HadoopAccessorService; import org.apache.oozie.service.LiteWorkflowStoreService; import org.apache.oozie.service.Services; @@ -951,8 +953,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { } // Define 'abc' token type in oozie-site - Configuration conf = Services.get().getConf(); - conf.set("oozie.credentials.credentialclasses", "abc=org.apache.oozie.action.hadoop.InsertTestToken"); + ConfigurationService.set("oozie.credentials.credentialclasses", "abc=org.apache.oozie.action.hadoop.InsertTestToken"); // Try to load the token after being defined in oozie-site; should work correctly credentialsConf = new JobConf(); @@ -1028,7 +1029,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { Assert.assertArrayEquals(new String[] { "java-action-executor" }, ae.getShareLibNames(context, new Element("java"), actionConf)); - Services.get().getConf().set("oozie.action.sharelib.for.java", "java-oozie-conf"); + ConfigurationService.set("oozie.action.sharelib.for.java", "java-oozie-conf"); Assert.assertArrayEquals(new String[] { "java-oozie-conf" }, ae.getShareLibNames(context, new Element("java"), actionConf)); @@ -1301,7 +1302,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { wfConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true); workflow.setConf(XmlUtils.prettyPrint(wfConf).toString()); - Services.get().getConf().set("oozie.action.sharelib.for.java", "java,hcat"); + ConfigurationService.set("oozie.action.sharelib.for.java", "java,hcat"); JavaActionExecutor ae = new JavaActionExecutor(); @@ -1339,7 +1340,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { wfConf.set("oozie.action.sharelib.for.java", "other,hcat"); workflow.setConf(XmlUtils.prettyPrint(wfConf).toString()); - Services.get().getConf().set("oozie.action.sharelib.for.java", "java"); + ConfigurationService.set("oozie.action.sharelib.for.java", "java"); ae = new JavaActionExecutor(); jobConf = ae.createBaseHadoopConf(context, eActionXml); @@ -1636,7 +1637,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals("false", conf.get("mapreduce.job.ubertask.enable")); // disable at oozie-site level for just the "test" action - Services.get().getConf().setBoolean("oozie.action.test.launcher.mapreduce.job.ubertask.enable", false); + ConfigurationService.setBoolean("oozie.action.test.launcher.mapreduce.job.ubertask.enable", false); JavaActionExecutor tjae = new JavaActionExecutor("test"); // default -- should not set @@ -1677,8 +1678,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertEquals("false", conf.get("mapreduce.job.ubertask.enable")); // disable at oozie-site level for all actions except for the "test" action - Services.get().getConf().setBoolean("oozie.action.test.launcher.mapreduce.job.ubertask.enable", true); - Services.get().getConf().setBoolean("oozie.action.launcher.mapreduce.job.ubertask.enable", false); + ConfigurationService.setBoolean("oozie.action.test.launcher.mapreduce.job.ubertask.enable", true); + ConfigurationService.setBoolean("oozie.action.launcher.mapreduce.job.ubertask.enable", false); // default -- should be true conf = new Configuration(false); @@ -2036,11 +2037,85 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { context = new Context(wf, action); launcherConf = new JobConf(); - launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml1, launcherConf); + launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml2, launcherConf); - // uber mode should be disabled since oozie.launcher.mapreduce.job.classloader=true + // uber mode should be disabled since oozie.launcher.mapreduce.user.classpath.first=true assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE)); -} + } + + public void testUpdateConfForTimeLineServiceEnabled() throws Exception { + Element actionXml = XmlUtils + .parseXml("<java>" + + "<job-tracker>" + + getJobTrackerUri() + + "</job-tracker>" + + "<name-node>" + + getNameNodeUri() + + "</name-node>" + + "<main-class>MAIN-CLASS</main-class>" + + "</java>"); + JavaActionExecutor ae = new JavaActionExecutor(); + XConfiguration protoConf = new XConfiguration(); + protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); + WorkflowJobBean wf = createBaseWorkflow(protoConf, "action"); + WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); + action.setType(ae.getType()); + Context context = new Context(wf, action); + JobConf actionConf = new JobConf(); + + // Test when server side setting is not enabled + JobConf launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, actionConf); + assertNull(launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED)); + + ConfigurationService.set("oozie.action.launcher." + JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED, "true"); + + // Test when server side setting is enabled but tez-site.xml is not in DistributedCache + launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, actionConf); + assertNull(launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED)); + + final Path tezSite = new Path("/tmp/tez-site.xml"); + final FSDataOutputStream out = getFileSystem().create(tezSite); + out.close(); + + // Test when server side setting is enabled and tez-site.xml is in DistributedCache + Element actionXmlWithTez = XmlUtils + .parseXml("<java>" + + "<job-tracker>" + + getJobTrackerUri() + + "</job-tracker>" + + "<name-node>" + + getNameNodeUri() + + "</name-node>" + + "<main-class>MAIN-CLASS</main-class>" + + "<file>" + tezSite + "</file>" + + "</java>"); + launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXmlWithTez, actionConf); + assertTrue(launcherConf.getBoolean(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED, false)); + + // Test when server side setting is enabled, tez-site.xml is in DistributedCache + // but user has disabled in action configuration + Element actionXmlATSDisabled = XmlUtils + .parseXml("<java>" + + "<job-tracker>" + + getJobTrackerUri() + + "</job-tracker>" + + "<name-node>" + + getNameNodeUri() + + "</name-node>" + + "<configuration>" + + "<property><name>oozie.launcher.yarn.timeline-service.enabled</name>" + + "<value>false</value></property>" + + "</configuration>" + + "<main-class>MAIN-CLASS</main-class>" + + "<file>" + tezSite + "</file>" + + "</java>"); + actionConf = ae.createBaseHadoopConf(context, actionXmlATSDisabled); + ae.setupActionConf(actionConf, context, actionXmlATSDisabled, new Path("hdfs:///tmp/workflow")); + launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXmlATSDisabled, actionConf); + assertFalse(launcherConf.getBoolean(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED, false)); + + getFileSystem().delete(tezSite, true); + } public void testAddToCache() throws Exception { JavaActionExecutor ae = new JavaActionExecutor(); @@ -2315,7 +2390,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { wfConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true); workflow.setConf(XmlUtils.prettyPrint(wfConf).toString()); - Services.get().getConf().set("oozie.action.sharelib.for.java", "java"); + ConfigurationService.set("oozie.action.sharelib.for.java", "java"); final RunningJob runningJob = submitAction(context); waitFor(60 * 1000, new Predicate() { http://git-wip-us.apache.org/repos/asf/oozie/blob/97b21af4/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index ccca437..f592299 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.2.0 release (trunk - unreleased) +OOZIE-2133 Support getting ATS delegation tokens for tez jobs (rohini) OOZIE-2131 Add flag to sqoop action to skip hbase delegation token generation (abec via ranter) OOZIE-2127 Add created time to RecoveryService WF queries (puru) OOZIE-2123 Disable launcher uber mode if classloader options are set (ryota)
