Author: daijy Date: Mon May 17 01:48:24 2010 New Revision: 944950 URL: http://svn.apache.org/viewvc?rev=944950&view=rev Log: PIG-1403: Make Pig work with remote HDFS in secure mode
Modified: hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestParser.java Modified: hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=944950&r1=944949&r2=944950&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt (original) +++ hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Mon May 17 01:48:24 2010 @@ -1381,35 +1381,37 @@ LogicalOperator LoadClause(LogicalPlan l if (absolutePath == null) { absolutePath = loFunc.relativeToAbsolutePath(filename, getCurrentDir(pigContext)); - // Get native host - String defaultFS = (String)pigContext.getProperties().get("fs.default.name"); - URI defaultFSURI = new URI(defaultFS); - String defaultHost = defaultFSURI.getHost(); - if (defaultHost==null) - defaultHost=""; - defaultHost = defaultHost.toLowerCase(); - - Set<String> remoteHosts = getRemoteHosts(absolutePath, defaultHost); - - String hdfsServersString = (String)pigContext.getProperties().get("mapreduce.job.hdfs-servers"); - if (hdfsServersString==null) hdfsServersString=""; - String hdfsServers[] = hdfsServersString.split(","); - - for (String remoteHost : remoteHosts) { - boolean existing = false; - for (String hdfsServer:hdfsServers) { - if (hdfsServer.equals(remoteHost)) - existing = true; - } - if (!existing) { - if (!hdfsServersString.isEmpty()) - hdfsServersString = hdfsServersString + ","; - hdfsServersString = hdfsServersString + remoteHost; - } - } - - if (!hdfsServersString.isEmpty()) - pigContext.getProperties().setProperty("mapreduce.job.hdfs-servers", hdfsServersString); + if (absolutePath!=null) { + // Get native host + String defaultFS = (String)pigContext.getProperties().get("fs.default.name"); + URI defaultFSURI = new URI(defaultFS); + String defaultHost = defaultFSURI.getHost(); + if (defaultHost==null) + defaultHost=""; + defaultHost = defaultHost.toLowerCase(); + + Set<String> remoteHosts = getRemoteHosts(absolutePath, defaultHost); + + String hdfsServersString = (String)pigContext.getProperties().get("mapreduce.job.hdfs-servers"); + if (hdfsServersString==null) hdfsServersString=""; + String hdfsServers[] = hdfsServersString.split(","); + + for (String remoteHost : remoteHosts) { + boolean existing = false; + for (String hdfsServer:hdfsServers) { + if (hdfsServer.equals(remoteHost)) + existing = true; + } + if (!existing) { + if (!hdfsServersString.isEmpty()) + hdfsServersString = hdfsServersString + ","; + hdfsServersString = hdfsServersString + remoteHost; + } + } + + if (!hdfsServersString.isEmpty()) + pigContext.getProperties().setProperty("mapreduce.job.hdfs-servers", hdfsServersString); + } fileNameMap.put(constructFileNameSignature(filename, funcSpec), absolutePath); } lo = new LOLoad(lp, new OperatorKey(scope, getNextId()), new FileSpec(absolutePath, funcSpec), Modified: hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestParser.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestParser.java?rev=944950&r1=944949&r2=944950&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestParser.java (original) +++ hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestParser.java Mon May 17 01:48:24 2010 @@ -92,30 +92,22 @@ protected final Log log = LogFactory.get public void testRemoteServerList() throws ExecException, IOException { try { Properties pigProperties = pigServer.getPigContext().getProperties(); - pigProperties.setProperty("fs.default.name", "hdfs://a.com:8020"); - + pigServer.registerQuery("a = load '/user/pig/1.txt';"); assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")==null); - pigServer.registerQuery("a = load 'hdfs://a.com/user/pig/1.txt';"); - assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")==null); - - pigServer.registerQuery("a = load 'har:///1.txt';"); - assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")==null); - pigServer.registerQuery("a = load 'hdfs://b.com/user/pig/1.txt';"); assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")!=null && pigProperties.getProperty("mapreduce.job.hdfs-servers").contains("hdfs://b.com")); - pigServer.registerQuery("a = load 'har://hdfs-c.com/user/pig/1.txt';"); + pigServer.registerQuery("a = load 'har://hdfs-c.com:8020/user/pig/1.txt';"); assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")!=null && - pigProperties.getProperty("mapreduce.job.hdfs-servers").contains("hdfs://c.com")); + pigProperties.getProperty("mapreduce.job.hdfs-servers").contains("hdfs://c.com:8020")); pigServer.registerQuery("a = load 'hdfs://d.com:8020/user/pig/1.txt';"); assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")!=null && pigProperties.getProperty("mapreduce.job.hdfs-servers").contains("hdfs://d.com:8020")); - } catch (IOException io) { } }