Repository: oozie Updated Branches: refs/heads/master 396fcc6c4 -> 1502633e7
OOZIE-2356 Add a way to enable/disable credentials in a workflow (rkanter) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/1502633e Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/1502633e Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/1502633e Branch: refs/heads/master Commit: 1502633e76c63c2da6ac055284d0b0eb4c4dc44a Parents: 396fcc6 Author: Robert Kanter <[email protected]> Authored: Wed Sep 9 10:33:04 2015 -0700 Committer: Robert Kanter <[email protected]> Committed: Wed Sep 9 10:33:04 2015 -0700 ---------------------------------------------------------------------- .../oozie/action/hadoop/JavaActionExecutor.java | 65 ++++++---- core/src/main/resources/oozie-default.xml | 8 ++ .../action/hadoop/TestJavaActionExecutor.java | 124 +++++++++++++++++++ .../site/twiki/DG_ActionAuthentication.twiki | 13 ++ release-log.txt | 1 + 5 files changed, 188 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/1502633e/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 0ea2eea..a6e7000 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -122,6 +122,7 @@ public class JavaActionExecutor extends ActionExecutor { public static final String CONF_HADOOP_YARN_UBER_MODE = "oozie.action.launcher." + HADOOP_YARN_UBER_MODE; public static final String HADOOP_JOB_CLASSLOADER = "mapreduce.job.classloader"; public static final String HADOOP_USER_CLASSPATH_FIRST = "mapreduce.user.classpath.first"; + public static final String OOZIE_CREDENTIALS_SKIP = "oozie.credentials.skip"; static { DISALLOWED_PROPERTIES.add(HADOOP_USER); @@ -1052,18 +1053,21 @@ public class JavaActionExecutor extends ActionExecutor { } // Setting the credential properties in launcher conf + JobConf credentialsConf = null; HashMap<String, CredentialsProperties> credentialsProperties = setCredentialPropertyToActionConf(context, action, actionConf); + if (credentialsProperties != null) { - // Adding if action need to set more credential tokens - JobConf credentialsConf = new JobConf(false); - XConfiguration.copy(actionConf, credentialsConf); - setCredentialTokens(credentialsConf, context, action, credentialsProperties); + // Adding if action need to set more credential tokens + credentialsConf = new JobConf(false); + XConfiguration.copy(actionConf, credentialsConf); + setCredentialTokens(credentialsConf, context, action, credentialsProperties); - // insert conf to action conf from credentialsConf - for (Entry<String, String> entry : credentialsConf) { - if (actionConf.get(entry.getKey()) == null) { - actionConf.set(entry.getKey(), entry.getValue()); + // insert conf to action conf from credentialsConf + for (Entry<String, String> entry : credentialsConf) { + if (actionConf.get(entry.getKey()) == null) { + actionConf.set(entry.getKey(), entry.getValue()); + } } } @@ -1098,7 +1102,7 @@ public class JavaActionExecutor extends ActionExecutor { launcherJobConf.getCredentials().addToken(HadoopAccessorService.MR_TOKEN_ALIAS, mrdt); // insert credentials tokens to launcher job conf if needed - if (needInjectCredentials()) { + if (needInjectCredentials() && credentialsConf != null) { for (Token<? extends TokenIdentifier> tk : credentialsConf.getCredentials().getAllTokens()) { Text fauxAlias = new Text(tk.getKind() + "_" + tk.getService()); LOG.debug("ADDING TOKEN: " + fauxAlias); @@ -1163,24 +1167,39 @@ public class JavaActionExecutor extends ActionExecutor { WorkflowAction action, Configuration actionConf) throws Exception { HashMap<String, CredentialsProperties> credPropertiesMap = null; if (context != null && action != null) { - credPropertiesMap = getActionCredentialsProperties(context, action); - if (credPropertiesMap != null) { - for (String key : credPropertiesMap.keySet()) { - CredentialsProperties prop = credPropertiesMap.get(key); - if (prop != null) { - LOG.debug("Credential Properties set for action : " + action.getId()); - for (String property : prop.getProperties().keySet()) { - actionConf.set(property, prop.getProperties().get(property)); - LOG.debug("property : '" + property + "', value : '" + prop.getProperties().get(property) + "'"); + if (!"true".equals(actionConf.get(OOZIE_CREDENTIALS_SKIP))) { + XConfiguration wfJobConf = null; + try { + wfJobConf = new XConfiguration(new StringReader(context.getWorkflow().getConf())); + } catch (IOException ioe) { + throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "It should never happen", + ioe.getMessage()); + } + if ("false".equals(actionConf.get(OOZIE_CREDENTIALS_SKIP)) || + !wfJobConf.getBoolean(OOZIE_CREDENTIALS_SKIP, ConfigurationService.getBoolean(OOZIE_CREDENTIALS_SKIP))) { + credPropertiesMap = getActionCredentialsProperties(context, action); + if (credPropertiesMap != null) { + for (String key : credPropertiesMap.keySet()) { + CredentialsProperties prop = credPropertiesMap.get(key); + if (prop != null) { + LOG.debug("Credential Properties set for action : " + action.getId()); + for (String property : prop.getProperties().keySet()) { + actionConf.set(property, prop.getProperties().get(property)); + LOG.debug("property : '" + property + "', value : '" + prop.getProperties().get(property) + + "'"); + } + } } + } else { + LOG.warn("No credential properties found for action : " + action.getId() + ", cred : " + action.getCred()); } + } else { + LOG.info("Skipping credentials (" + OOZIE_CREDENTIALS_SKIP + "=true)"); } + } else { + LOG.info("Skipping credentials (" + OOZIE_CREDENTIALS_SKIP + "=true)"); } - else { - LOG.warn("No credential properties found for action : " + action.getId() + ", cred : " + action.getCred()); - } - } - else { + } else { LOG.warn("context or action is null"); } return credPropertiesMap; http://git-wip-us.apache.org/repos/asf/oozie/blob/1502633e/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 8a0bc3b..4b9a0bc 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -1932,6 +1932,14 @@ A list of credential class mapping for CredentialsProvider </description> </property> + <property> + <name>oozie.credentials.skip</name> + <value>false</value> + <description> + This determines if Oozie should skip getting credentials from the credential providers. This can be overwritten at a + job-level or action-level. + </description> + </property> <property> <name>oozie.actions.main.classnames</name> http://git-wip-us.apache.org/repos/asf/oozie/blob/1502633e/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index 8b867bb..9be2c51 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -29,8 +29,10 @@ import java.io.Writer; import java.net.URI; import java.text.SimpleDateFormat; import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.Map; import java.util.Properties; import org.apache.hadoop.conf.Configuration; @@ -938,6 +940,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { HashMap<String, CredentialsProperties> credProperties = ae.setCredentialPropertyToActionConf(context, action, actionConf); + assertNotNull(credProperties); CredentialsProperties prop = credProperties.get("abcname"); assertEquals("value1", prop.getProperties().get("property1")); assertEquals("value2", prop.getProperties().get("property2")); @@ -969,7 +972,123 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { assertNotNull(tk); } + public void testCredentialsSkip() throws Exception { + // Try setting oozie.credentials.skip at different levels, and verifying the correct behavior + // oozie-site: false -- job-level: null -- action-level: null + _testCredentialsSkip(false, null, null, true); + + // oozie-site: false -- job-level: null -- action-level: false + _testCredentialsSkip(false, null, "false", true); + + // oozie-site: false -- job-level: null -- action-level: true + _testCredentialsSkip(false, null, "true", false); + + // oozie-site: false -- job-level: false -- action-level: null + _testCredentialsSkip(false, "false", null, true); + + // oozie-site: false -- job-level: false -- action-level: false + _testCredentialsSkip(false, "false", "false", true); + + // oozie-site: false -- job-level: false -- action-level: true + _testCredentialsSkip(false, "false", "true", false); + + // oozie-site: false -- job-level: true -- action-level: null + _testCredentialsSkip(false, "true", null, false); + + // oozie-site: false -- job-level: true -- action-level: false + _testCredentialsSkip(false, "true", "false", true); + + // oozie-site: false -- job-level: true -- action-level: true + _testCredentialsSkip(false, "true", "true", false); + + // oozie-site: true -- job-level: null -- action-level: null + _testCredentialsSkip(true, null, null, false); + + // oozie-site: true -- job-level: null -- action-level: false + _testCredentialsSkip(true, null, "false", true); + + // oozie-site: true -- job-level: null -- action-level: true + _testCredentialsSkip(true, null, "true", false); + + // oozie-site: true -- job-level: false -- action-level: null + _testCredentialsSkip(true, "false", null, true); + + // oozie-site: true -- job-level: false -- action-level: false + _testCredentialsSkip(true, "false", "false", true); + + // oozie-site: true -- job-level: false -- action-level: true + _testCredentialsSkip(true, "false", "true", false); + + // oozie-site: true -- job-level: true -- action-level: null + _testCredentialsSkip(true, "true", null, false); + + // oozie-site: true -- job-level: true -- action-level: false + _testCredentialsSkip(true, "true", "false", true); + + // oozie-site: true -- job-level: true -- action-level: true + _testCredentialsSkip(true, "true", "true", false); + } + + private void _testCredentialsSkip(boolean skipSite, String skipJob, String skipAction, boolean expectingTokens) + throws Exception { + String actionLevelSkipConf = (skipAction == null) ? "" : + "<property><name>oozie.credentials.skip</name><value>" + skipAction + "</value></property>"; + String actionxml = "<pig>" + "<job-tracker>${jobTracker}</job-tracker>" + "<name-node>${nameNode}</name-node>" + + "<prepare>" + "<delete path='outputdir' />" + "</prepare>" + "<configuration>" + "<property>" + + "<name>mapred.compress.map.output</name>" + "<value>true</value>" + "</property>" + "<property>" + + "<name>mapred.job.queue.name</name>" + "<value>${queueName}</value>" + "</property>" + actionLevelSkipConf + + "</configuration>" + "<script>org/apache/oozie/examples/pig/id.pig</script>" + + "<param>INPUT=${inputDir}</param>" + "<param>OUTPUT=${outputDir}/pig-output</param>" + "</pig>"; + String workflowXml = "<workflow-app xmlns='uri:oozie:workflow:0.2.5' name='pig-wf'>" + "<credentials>" + + "<credential name='abcname' type='abc'>" + "<property>" + "<name>property1</name>" + + "<value>value1</value>" + "</property>" + "<property>" + "<name>property2</name>" + + "<value>value2</value>" + "</property>" + "<property>" + "<name>${property3}</name>" + + "<value>${value3}</value>" + "</property>" + "</credential>" + "</credentials>" + + "<start to='pig1' />" + "<action name='pig1' cred='abcname'>" + actionxml + + "<ok to='end' />" + "<error to='fail' />" + "</action>" + "<kill name='fail'>" + + "<message>Pig failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>" + "</kill>" + + "<end name='end' />" + "</workflow-app>"; + + JavaActionExecutor ae = new JavaActionExecutor(); + WorkflowJobBean wfBean = addRecordToWfJobTable("test1", workflowXml, + (skipJob == null) ? null : Collections.singletonMap("oozie.credentials.skip", skipJob)); + WorkflowActionBean action = (WorkflowActionBean) wfBean.getActions().get(0); + action.setType(ae.getType()); + action.setCred("abcname"); + action.setConf(actionxml); + Context context = new Context(wfBean, action); + + Element actionXmlconf = XmlUtils.parseXml(action.getConf()); + // action job configuration + Configuration actionConf = ae.createBaseHadoopConf(context, actionXmlconf); + actionConf = ae.setupActionConf(actionConf, context, actionXmlconf, new Path("/tmp/foo")); + + // Define 'abc' token type in oozie-site + ConfigurationService.set("oozie.credentials.credentialclasses", "abc=org.apache.oozie.action.hadoop.InsertTestToken"); + ConfigurationService.setBoolean("oozie.credentials.skip", skipSite); + + // Setting the credential properties in launcher conf + HashMap<String, CredentialsProperties> credProperties = ae.setCredentialPropertyToActionConf(context, + action, actionConf); + + // Try to load the token without it being defined in oozie-site; should get an exception + JobConf credentialsConf = new JobConf(); + Configuration launcherConf = ae.createBaseHadoopConf(context, actionXmlconf); + XConfiguration.copy(launcherConf, credentialsConf); + ae.setCredentialTokens(credentialsConf, context, action, credProperties); + Token<? extends TokenIdentifier> tk = credentialsConf.getCredentials().getToken(new Text("ABC Token")); + if (expectingTokens) { + assertNotNull(tk); + } else { + assertNull(tk); + } + } + private WorkflowJobBean addRecordToWfJobTable(String wfId, String wfxml) throws Exception { + return addRecordToWfJobTable(wfId, wfxml, null); + } + + private WorkflowJobBean addRecordToWfJobTable(String wfId, String wfxml, Map<String, String> otherProps) throws Exception { WorkflowApp app = new LiteWorkflowApp("testApp", wfxml, new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "start")). addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class)); @@ -980,6 +1099,11 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { conf.set(OozieClient.USER_NAME, getTestUser()); conf.set("property3", "prop3"); conf.set("value3", "val3"); + if (otherProps != null) { + for (Map.Entry<String, String> ent : otherProps.entrySet()) { + conf.set(ent.getKey(), ent.getValue()); + } + } WorkflowJobBean wfBean = createWorkflow(app, conf, "auth"); wfBean.setId(wfId); http://git-wip-us.apache.org/repos/asf/oozie/blob/1502633e/docs/src/site/twiki/DG_ActionAuthentication.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_ActionAuthentication.twiki b/docs/src/site/twiki/DG_ActionAuthentication.twiki index 357a5f9..e980e02 100644 --- a/docs/src/site/twiki/DG_ActionAuthentication.twiki +++ b/docs/src/site/twiki/DG_ActionAuthentication.twiki @@ -88,6 +88,19 @@ so that Oozie will include these credentials with the action. You can include m a comma-separated list of =credential= names. And finally, the HCatCredentials required two properties (the metastore URI and principal), which we also specified. +Adding the =credentials= section to a workflow and referencing it in an action will make Oozie always try to obtain that delegation +token. Ordinarily, this would mean that you cannot re-use this workflow in a non-secure cluster without editing it because trying +to obtain the delegation token will likely fail. However, you can tell Oozie to ignore the =credentials= for a workflow by setting +the job-level property =oozie.credentials.skip= to =true=; this will allow you to use the same workflow.xml in a secure and +non-secure cluster by simply changing the job-level property at runtime. If omitted or set to =false=, Oozie will handle +the =credentials= section normally. In addition, you can also set this property at the action-level or server-level to skip getting +credentials for just that action or for all workflows, respectively. The order of priority is this: + + 1. =oozie.credentials.skip= in the =configuration= section of an action, if set + 1. =oozie.credentials.skip= in the job.properties for a workflow, if set + 1. =oozie.credentials.skip= in oozie-site.xml for all workflows, if set + 1. (don't skip) + ---++ Built-in Credentials Implementations Oozie currently comes with the following Credentials implementations: http://git-wip-us.apache.org/repos/asf/oozie/blob/1502633e/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 0bd450e..bf84976 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2356 Add a way to enable/disable credentials in a workflow (rkanter) OOZIE-2355 Hive2 Action doesn't pass along oozie configs to jobconf (rkanter) OOZIE-2318 Provide better solution for specifying SSL truststore to Oozie Client (rkanter) OOZIE-2344 Enabling 'oozie.action.jobinfo.enable' doesn't inject the job information into the map/reduce job's configuration. (akshayrai09 via rkanter)
