Author: daijy
Date: Mon May 17 01:48:24 2010
New Revision: 944950

URL: http://svn.apache.org/viewvc?rev=944950&view=rev
Log:
PIG-1403: Make Pig work with remote HDFS in secure mode

Modified:
    
hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
    hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestParser.java

Modified: 
hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=944950&r1=944949&r2=944950&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
 (original)
+++ 
hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
 Mon May 17 01:48:24 2010
@@ -1381,35 +1381,37 @@ LogicalOperator LoadClause(LogicalPlan l
             if (absolutePath == null) {
                 absolutePath = loFunc.relativeToAbsolutePath(filename, 
getCurrentDir(pigContext));
                 
-                // Get native host
-                String defaultFS = 
(String)pigContext.getProperties().get("fs.default.name");
-                URI defaultFSURI = new URI(defaultFS);
-                String defaultHost = defaultFSURI.getHost();
-                if (defaultHost==null)
-                    defaultHost="";
-                defaultHost = defaultHost.toLowerCase();
-
-                Set<String> remoteHosts = getRemoteHosts(absolutePath, 
defaultHost);
-                
-                String hdfsServersString = 
(String)pigContext.getProperties().get("mapreduce.job.hdfs-servers");
-                if (hdfsServersString==null) hdfsServersString="";
-                String hdfsServers[] = hdfsServersString.split(",");
-                
-                for (String remoteHost : remoteHosts) {
-                    boolean existing = false;
-                    for (String hdfsServer:hdfsServers) {
-                        if (hdfsServer.equals(remoteHost))
-                            existing = true;
-                    }
-                    if (!existing) {
-                        if (!hdfsServersString.isEmpty())
-                            hdfsServersString = hdfsServersString + ",";
-                        hdfsServersString = hdfsServersString + remoteHost;
-                    }
-                }
-
-                if (!hdfsServersString.isEmpty())
-                    
pigContext.getProperties().setProperty("mapreduce.job.hdfs-servers", 
hdfsServersString);
+                if (absolutePath!=null) {
+                       // Get native host
+                       String defaultFS = 
(String)pigContext.getProperties().get("fs.default.name");
+                       URI defaultFSURI = new URI(defaultFS);
+                       String defaultHost = defaultFSURI.getHost();
+                       if (defaultHost==null)
+                           defaultHost="";
+                       defaultHost = defaultHost.toLowerCase();
+       
+                       Set<String> remoteHosts = getRemoteHosts(absolutePath, 
defaultHost);
+                       
+                       String hdfsServersString = 
(String)pigContext.getProperties().get("mapreduce.job.hdfs-servers");
+                       if (hdfsServersString==null) hdfsServersString="";
+                       String hdfsServers[] = hdfsServersString.split(",");
+                       
+                       for (String remoteHost : remoteHosts) {
+                           boolean existing = false;
+                           for (String hdfsServer:hdfsServers) {
+                               if (hdfsServer.equals(remoteHost))
+                                   existing = true;
+                           }
+                           if (!existing) {
+                               if (!hdfsServersString.isEmpty())
+                                   hdfsServersString = hdfsServersString + ",";
+                               hdfsServersString = hdfsServersString + 
remoteHost;
+                           }
+                       }
+       
+                       if (!hdfsServersString.isEmpty())
+                           
pigContext.getProperties().setProperty("mapreduce.job.hdfs-servers", 
hdfsServersString);
+                   }
                 fileNameMap.put(constructFileNameSignature(filename, 
funcSpec), absolutePath);
             }
             lo = new LOLoad(lp, new OperatorKey(scope, getNextId()), new 
FileSpec(absolutePath, funcSpec),

Modified: 
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestParser.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestParser.java?rev=944950&r1=944949&r2=944950&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestParser.java 
(original)
+++ hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestParser.java Mon 
May 17 01:48:24 2010
@@ -92,30 +92,22 @@ protected final Log log = LogFactory.get
     public void testRemoteServerList() throws ExecException, IOException {
         try {
             Properties pigProperties = 
pigServer.getPigContext().getProperties();
-            pigProperties.setProperty("fs.default.name", "hdfs://a.com:8020");
-            
+
             pigServer.registerQuery("a = load '/user/pig/1.txt';");
             
assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")==null);
             
-            pigServer.registerQuery("a = load 'hdfs://a.com/user/pig/1.txt';");
-            
assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")==null);
-            
-            pigServer.registerQuery("a = load 'har:///1.txt';");
-            
assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")==null);
-            
             pigServer.registerQuery("a = load 'hdfs://b.com/user/pig/1.txt';");
             
assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")!=null &&
                     
pigProperties.getProperty("mapreduce.job.hdfs-servers").contains("hdfs://b.com"));
             
-            pigServer.registerQuery("a = load 
'har://hdfs-c.com/user/pig/1.txt';");
+            pigServer.registerQuery("a = load 
'har://hdfs-c.com:8020/user/pig/1.txt';");
             
assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")!=null &&
-                    
pigProperties.getProperty("mapreduce.job.hdfs-servers").contains("hdfs://c.com"));
+                    
pigProperties.getProperty("mapreduce.job.hdfs-servers").contains("hdfs://c.com:8020"));
             
             pigServer.registerQuery("a = load 
'hdfs://d.com:8020/user/pig/1.txt';");
             
assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")!=null &&
                     
pigProperties.getProperty("mapreduce.job.hdfs-servers").contains("hdfs://d.com:8020"));
 
-
         } catch (IOException io) {
         }
     }


Reply via email to