Author: hashutosh
Date: Thu Sep 8 18:23:00 2011
New Revision: 1166837
URL: http://svn.apache.org/viewvc?rev=1166837&view=rev
Log:
HCATALOG-94 : Handle the job tracker tokens properly in har jobs
Modified:
incubator/hcatalog/branches/branch-0.2/CHANGES.txt
incubator/hcatalog/branches/branch-0.2/src/java/org/apache/hcatalog/mapreduce/HCatOutputCommitter.java
incubator/hcatalog/branches/branch-0.2/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java
Modified: incubator/hcatalog/branches/branch-0.2/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.2/CHANGES.txt?rev=1166837&r1=1166836&r2=1166837&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.2/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.2/CHANGES.txt Thu Sep 8 18:23:00 2011
@@ -33,9 +33,11 @@ Release 0.2.0 - Unreleased
(Krishna Kumar via macyang)
IMPROVEMENTS
- HCAT-43. hcat.sh does not pick up hcatalog.jar correctly (macyang via
hashutosh)
+ HCAT-94. Handle the job tracker tokens properly in har jobs (khorgath via
hashutosh)
+
+ HCAT-43. hcat.sh does not pick up hcatalog.jar correctly (macyang via
hashutosh)
- HCAT-91. hcat client not setting the config properly (hashutosh)
+ HCAT-91. hcat client not setting the config properly (hashutosh)
HCAT-66. HCatalog 0.2.0 Documentation (chandec via hashutosh)
Modified:
incubator/hcatalog/branches/branch-0.2/src/java/org/apache/hcatalog/mapreduce/HCatOutputCommitter.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.2/src/java/org/apache/hcatalog/mapreduce/HCatOutputCommitter.java?rev=1166837&r1=1166836&r2=1166837&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.2/src/java/org/apache/hcatalog/mapreduce/HCatOutputCommitter.java
(original)
+++
incubator/hcatalog/branches/branch-0.2/src/java/org/apache/hcatalog/mapreduce/HCatOutputCommitter.java
Thu Sep 8 18:23:00 2011
@@ -171,13 +171,13 @@ public class HCatOutputCommitter extends
(HCatConstants.HCAT_KEY_TOKEN_SIGNATURE) != null) {
client.cancelDelegationToken(tokenStrForm);
}
-
- String jcTokenStrForm =
jobContext.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_STRFORM);
- String jcTokenSignature =
jobContext.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_SIGNATURE);
- if(jcTokenStrForm != null && jcTokenSignature != null) {
-
HCatUtil.cancelJobTrackerDelegationToken(tokenStrForm,jcTokenSignature);
+ if (harProcessor.isEnabled()){
+ String jcTokenStrForm =
jobContext.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_STRFORM);
+ String jcTokenSignature =
jobContext.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_SIGNATURE);
+ if(jcTokenStrForm != null && jcTokenSignature != null) {
+
HCatUtil.cancelJobTrackerDelegationToken(tokenStrForm,jcTokenSignature);
+ }
}
-
} catch(Exception e) {
if( e instanceof HCatException ) {
throw (HCatException) e;
@@ -376,12 +376,14 @@ public class HCatOutputCommitter extends
client.cancelDelegationToken(tokenStrForm);
}
- String jcTokenStrForm =
-
context.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_STRFORM);
- String jcTokenSignature =
-
context.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_SIGNATURE);
- if(jcTokenStrForm != null && jcTokenSignature != null) {
-
HCatUtil.cancelJobTrackerDelegationToken(tokenStrForm,jcTokenSignature);
+ if (harProcessor.isEnabled()){
+ String jcTokenStrForm =
+
context.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_STRFORM);
+ String jcTokenSignature =
+
context.getConfiguration().get(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_SIGNATURE);
+ if(jcTokenStrForm != null && jcTokenSignature != null) {
+
HCatUtil.cancelJobTrackerDelegationToken(tokenStrForm,jcTokenSignature);
+ }
}
} catch (Exception e) {
Modified:
incubator/hcatalog/branches/branch-0.2/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.2/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java?rev=1166837&r1=1166836&r2=1166837&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.2/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java
(original)
+++
incubator/hcatalog/branches/branch-0.2/src/java/org/apache/hcatalog/mapreduce/HCatOutputFormat.java
Thu Sep 8 18:23:00 2011
@@ -247,9 +247,11 @@ public class HCatOutputFormat extends HC
}
String jcTokenSignature = "jc."+tokenSignature;
- if(tokenMap.get(jcTokenSignature) == null) {
- tokenMap.put(jcTokenSignature,
-
HCatUtil.getJobTrackerDelegationToken(conf,ugi.getUserName()));
+ if (harRequested){
+ if(tokenMap.get(jcTokenSignature) == null) {
+ tokenMap.put(jcTokenSignature,
+
HCatUtil.getJobTrackerDelegationToken(conf,ugi.getUserName()));
+ }
}
job.getCredentials().addToken(new Text(ugi.getUserName() +
tokenSignature),
@@ -257,15 +259,20 @@ public class HCatOutputFormat extends HC
// this will be used by the outputcommitter to pass on to the
metastore client
// which in turn will pass on to the TokenSelector so that it can
select
// the right token.
- job.getCredentials().addToken(new Text(ugi.getUserName() +
jcTokenSignature),
- tokenMap.get(jcTokenSignature));
-
job.getConfiguration().set(HCatConstants.HCAT_KEY_TOKEN_SIGNATURE,
tokenSignature);
-
job.getConfiguration().set(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_SIGNATURE,
jcTokenSignature);
-
job.getConfiguration().set(HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_STRFORM,
tokenMap.get(jcTokenSignature).encodeToUrlString());
-// LOG.info("Set hive dt["+tokenSignature+"]");
-// LOG.info("Set jt dt["+jcTokenSignature+"]");
+ if (harRequested){
+ job.getCredentials().addToken(new Text(ugi.getUserName() +
jcTokenSignature),
+ tokenMap.get(jcTokenSignature));
+
+ job.getConfiguration().set(
+ HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_SIGNATURE,
jcTokenSignature);
+ job.getConfiguration().set(
+ HCatConstants.HCAT_KEY_JOBCLIENT_TOKEN_STRFORM,
+ tokenMap.get(jcTokenSignature).encodeToUrlString());
+ // LOG.info("Set hive dt["+tokenSignature+"]");
+ // LOG.info("Set jt dt["+jcTokenSignature+"]");
+ }
}
}
} catch(Exception e) {
