[jira] [Comment Edited] (TEZ-1611) Change DataSource/Sink to be able to supply URIs for credentials

[Bikas Saha (JIRA)]

    [ 
https://issues.apache.org/jira/browse/TEZ-1611?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14148115#comment-14148115
 ] 

Bikas Saha edited comment on TEZ-1611 at 9/25/14 6:37 PM:
----------------------------------------------------------

{code}+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java     
                                                 
@@ -83,6 +83,7 @@ import org.apache.tez.common.security.JobTokenSecretManager;  
                                           
 import org.apache.tez.common.security.TokenCache;                              
                                           
 import org.apache.tez.dag.api.DAG;
 import org.apache.tez.dag.api.DagTypeConverters;
+import org.apache.tez.dag.api.DataSinkDescriptor;
 import org.apache.tez.dag.api.DataSourceDescriptor;
 import org.apache.tez.dag.api.SessionNotRunning;
 import org.apache.tez.dag.api.TezConfiguration;
@@ -361,6 +362,10 @@ public class TezClientUtils {
         for (DataSourceDescriptor dataSource : dataSources) {
           addFileSystemCredentialsFromURIs(dataSource.getURIsForCredentials(), 
dagCredentials, conf);
         }
+        List<DataSinkDescriptor> dataSinks = v.getDataSinks();
+        for (DataSinkDescriptor dataSink : dataSinks) {
+          addFileSystemCredentialsFromURIs(dataSink.getURIsForCredentials(), 
dagCredentials, conf);
+        }
       }

       for (LocalResource lr: dag.getTaskLocalFiles().values()) {
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java 
b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex
index 4f31090..04acdaf 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
@@ -446,7 +446,8 @@ public class Vertex {
     return dataSources;
   }

-  List<DataSinkDescriptor> getDataSinks() {
+  @Private
+  public List<DataSinkDescriptor> getDataSinks() {
     return dataSinks;
   }
{code}


was (Author: bikassaha):
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java           
                                           
@@ -83,6 +83,7 @@ import org.apache.tez.common.security.JobTokenSecretManager;  
                                           
 import org.apache.tez.common.security.TokenCache;                              
                                           
 import org.apache.tez.dag.api.DAG;
 import org.apache.tez.dag.api.DagTypeConverters;
+import org.apache.tez.dag.api.DataSinkDescriptor;
 import org.apache.tez.dag.api.DataSourceDescriptor;
 import org.apache.tez.dag.api.SessionNotRunning;
 import org.apache.tez.dag.api.TezConfiguration;
@@ -361,6 +362,10 @@ public class TezClientUtils {
         for (DataSourceDescriptor dataSource : dataSources) {
           addFileSystemCredentialsFromURIs(dataSource.getURIsForCredentials(), 
dagCredentials, conf);
         }
+        List<DataSinkDescriptor> dataSinks = v.getDataSinks();
+        for (DataSinkDescriptor dataSink : dataSinks) {
+          addFileSystemCredentialsFromURIs(dataSink.getURIsForCredentials(), 
dagCredentials, conf);
+        }
       }

       for (LocalResource lr: dag.getTaskLocalFiles().values()) {
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java 
b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex
index 4f31090..04acdaf 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
@@ -446,7 +446,8 @@ public class Vertex {
     return dataSources;
   }

-  List<DataSinkDescriptor> getDataSinks() {
+  @Private
+  public List<DataSinkDescriptor> getDataSinks() {
     return dataSinks;
   }


> Change DataSource/Sink to be able to supply URIs for credentials
> ----------------------------------------------------------------
>
>                 Key: TEZ-1611
>                 URL: https://issues.apache.org/jira/browse/TEZ-1611
>             Project: Apache Tez
>          Issue Type: Bug
>            Reporter: Bikas Saha
>            Assignee: Bikas Saha
>         Attachments: TEZ-1611.1.patch, TEZ-1611.2.patch
>
>
> This way commonly used credentials (HDFS) can be obtained globally for the 
> DAG and take advantage of previously cached credentials where appropriate. 
> This would reduce round trips to the NN. The existing method to get 
> credentials will still be in place for other credentials that we cannot 
> obtain automatically via URIs



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)