Author: virag
Date: Wed Jan 2 19:51:46 2013
New Revision: 1427979
URL: http://svn.apache.org/viewvc?rev=1427979&view=rev
Log:
OOZIE-1148 Set the renewer correctly for JT/RM delegation tokens (rohini via
virag)
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
oozie/trunk/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
oozie/trunk/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
oozie/trunk/release-log.txt
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java?rev=1427979&r1=1427978&r2=1427979&view=diff
==============================================================================
---
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
(original)
+++
oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
Wed Jan 2 19:51:46 2013
@@ -46,7 +46,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AccessControlException;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobID;
@@ -729,8 +728,9 @@ public class JavaActionExecutor extends
XLog.getLog(getClass()).debug("Submitting the job through Job
Client for action " + action.getId());
// setting up propagation of the delegation token.
- Token<DelegationTokenIdentifier> mrdt =
jobClient.getDelegationToken(new
Text(HadoopAccessorService.MR_DELEGATION_TOKEN));
- launcherJobConf.getCredentials().addToken(new
Text(HadoopAccessorService.MR_DELEGATION_TOKEN), mrdt);
+ Token<DelegationTokenIdentifier> mrdt =
jobClient.getDelegationToken(HadoopAccessorService
+ .getMRDelegationTokenRenewer(launcherJobConf));
+
launcherJobConf.getCredentials().addToken(HadoopAccessorService.MR_TOKEN_ALIAS,
mrdt);
// insert credentials tokens to launcher job conf if needed
if (needInjectCredentials()) {
Modified:
oozie/trunk/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java?rev=1427979&r1=1427978&r2=1427979&view=diff
==============================================================================
---
oozie/trunk/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
(original)
+++
oozie/trunk/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
Wed Jan 2 19:51:46 2013
@@ -55,6 +55,8 @@ import java.util.concurrent.ConcurrentMa
*/
public class HadoopAccessorService implements Service {
+ private static XLog LOG = XLog.getLog(HadoopAccessorService.class);
+
public static final String CONF_PREFIX = Service.CONF_PREFIX +
"HadoopAccessorService.";
public static final String JOB_TRACKER_WHITELIST = CONF_PREFIX +
"jobTracker.whitelist";
public static final String NAME_NODE_WHITELIST = CONF_PREFIX +
"nameNode.whitelist";
@@ -63,9 +65,14 @@ public class HadoopAccessorService imple
public static final String KERBEROS_AUTH_ENABLED = CONF_PREFIX +
"kerberos.enabled";
public static final String KERBEROS_KEYTAB = CONF_PREFIX + "keytab.file";
public static final String KERBEROS_PRINCIPAL = CONF_PREFIX +
"kerberos.principal";
- public static final String MR_DELEGATION_TOKEN = "oozie mr token";
+ public static final Text MR_TOKEN_ALIAS = new Text("oozie mr token");
private static final String OOZIE_HADOOP_ACCESSOR_SERVICE_CREATED =
"oozie.HadoopAccessorService.created";
+ /** The Kerberos principal for the job tracker.*/
+ private static final String JT_PRINCIPAL =
"mapreduce.jobtracker.kerberos.principal";
+ /** The Kerberos principal for the resource manager.*/
+ private static final String RM_PRINCIPAL =
"yarn.resourcemanager.principal";
+ private static final Map<String, Text> mrTokenRenewers = new
HashMap<String, Text>();
private Set<String> jobTrackerWhitelist = new HashSet<String>();
private Set<String> nameNodeWhitelist = new HashSet<String>();
@@ -362,8 +369,8 @@ public class HadoopAccessorService imple
return new JobClient(conf);
}
});
- Token<DelegationTokenIdentifier> mrdt =
jobClient.getDelegationToken(new Text(MR_DELEGATION_TOKEN));
- conf.getCredentials().addToken(new Text(MR_DELEGATION_TOKEN),
mrdt);
+ Token<DelegationTokenIdentifier> mrdt =
jobClient.getDelegationToken(getMRDelegationTokenRenewer(conf));
+ conf.getCredentials().addToken(MR_TOKEN_ALIAS, mrdt);
return jobClient;
}
catch (InterruptedException ex) {
@@ -449,6 +456,26 @@ public class HadoopAccessorService imple
}
}
+ public static Text getMRDelegationTokenRenewer(JobConf jobConf) {
+ // Getting renewer correctly for JT principal also though JT in hadoop
1.x does not have
+ // support for renewing/cancelling tokens
+ String servicePrincipal = jobConf.get(RM_PRINCIPAL,
jobConf.get(JT_PRINCIPAL));
+ Text renewer;
+ if (servicePrincipal != null) { // secure cluster
+ renewer = mrTokenRenewers.get(servicePrincipal);
+ if (renewer == null) {
+ // Remove host and domain
+ renewer = new Text(servicePrincipal.split("[/@]")[0]);
+ LOG.info("Delegation Token Renewer for " + servicePrincipal +
" is " + renewer);
+ mrTokenRenewers.put(servicePrincipal, renewer);
+ }
+ }
+ else {
+ renewer = MR_TOKEN_ALIAS; //Doesn't matter what we pass as renewer
+ }
+ return renewer;
+ }
+
public void addFileToClassPath(String user, final Path file, final
Configuration conf)
throws IOException {
ParamChecker.notEmpty(user, "user");
Modified:
oozie/trunk/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
URL:
http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java?rev=1427979&r1=1427978&r2=1427979&view=diff
==============================================================================
---
oozie/trunk/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
(original)
+++
oozie/trunk/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
Wed Jan 2 19:51:46 2013
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,6 +21,7 @@ import org.apache.oozie.test.XTestCase;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.Text;
import org.apache.oozie.util.IOUtils;
import java.io.File;
@@ -113,4 +114,14 @@ public class TestHadoopAccessorService e
}
}
+ public void testGetMRDelegationTokenRenewer() throws Exception {
+ JobConf jobConf = new JobConf();
+ assertEquals(new Text("oozie mr token"),
HadoopAccessorService.getMRDelegationTokenRenewer(jobConf));
+ jobConf.set("mapreduce.jobtracker.kerberos.principal",
"mapred/[email protected]");
+ assertEquals(new Text("mapred"),
HadoopAccessorService.getMRDelegationTokenRenewer(jobConf));
+ jobConf = new JobConf();
+ jobConf.set("yarn.resourcemanager.principal",
"rm/[email protected]");
+ assertEquals(new Text("rm"),
HadoopAccessorService.getMRDelegationTokenRenewer(jobConf));
+ }
+
}
Modified: oozie/trunk/release-log.txt
URL:
http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1427979&r1=1427978&r2=1427979&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Wed Jan 2 19:51:46 2013
@@ -69,6 +69,7 @@ OOZIE-944 Implement Workflow Generator U
-- Oozie 3.3.1 (unreleased)
+OOZIE-1148 Set the renewer correctly for JT/RM delegation tokens (rohini via
virag)
OOZIE-1147 HCatCredentialHelper uses the wrong API for getDelegationToken
(rohini via virag)
OOZIE-1149 Update 3.3 branch POM's to 3.3.1-SNAPSHOT (virag)
OOZIE-1139 bump up Tomcat version to 6.0.36 (tucu via rkanter)
