[incubator-gobblin] branch master updated: [GOBBLIN-1286] Added support to fetch multiple DFS tokens for HDFS federation

[suvasude]

This is an automated email from the ASF dual-hosted git repository.

suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new 268d608  [GOBBLIN-1286] Added support to fetch multiple DFS tokens for 
HDFS federation
268d608 is described below

commit 268d6084c1ef07e6be44f7499e10af81d9877190
Author: Sandeep Devaiah <sdeva...@linkedin.com>
AuthorDate: Fri Oct 16 06:39:29 2020 -0700

    [GOBBLIN-1286] Added support to fetch multiple DFS tokens for HDFS 
federation
    
    Closes #3125 from sandy143651/master
---
 .../org/apache/gobblin/util/hadoop/TokenUtils.java     | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git 
a/gobblin-utility/src/main/java/org/apache/gobblin/util/hadoop/TokenUtils.java 
b/gobblin-utility/src/main/java/org/apache/gobblin/util/hadoop/TokenUtils.java
index 8c70a13..fd6463d 100644
--- 
a/gobblin-utility/src/main/java/org/apache/gobblin/util/hadoop/TokenUtils.java
+++ 
b/gobblin-utility/src/main/java/org/apache/gobblin/util/hadoop/TokenUtils.java
@@ -321,17 +321,15 @@ public class TokenUtils {
   private static void getHdfsToken(Configuration conf, Credentials cred) 
throws IOException {
     FileSystem fs = FileSystem.get(conf);
     LOG.info("Getting DFS token from " + fs.getUri());
-    Token<?> fsToken = fs.getDelegationToken(getMRTokenRenewerInternal(new 
JobConf()).toString());
-    if (fsToken == null) {
-      LOG.error("Failed to fetch DFS token for ");
-      throw new IOException("Failed to fetch DFS token.");
+    String renewer = getMRTokenRenewerInternal(new JobConf()).toString();
+    Token<?>[] fsTokens = fs.addDelegationTokens(renewer, cred);
+    for(int i = 0; i < fsTokens.length; i++) {
+      Token<?> token = fsTokens[i];
+      String message =
+          String.format("DFS token fetched from namenode, token kind: %s, 
token service %s", token.getKind(),
+              token.getService());
+      LOG.info(message);
     }
-    LOG.info("Created DFS token: " + fsToken.toString());
-    LOG.info("Token kind: " + fsToken.getKind());
-    LOG.info("Token id: " + Arrays.toString(fsToken.getIdentifier()));
-    LOG.info("Token service: " + fsToken.getService());
-
-    cred.addToken(fsToken.getService(), fsToken);
   }
 
   private static void getOtherNamenodesToken(List<String> otherNamenodes, 
Configuration conf, Credentials cred)