Author: pradeepkth Date: Tue Feb 16 18:42:14 2010 New Revision: 910645 URL: http://svn.apache.org/viewvc?rev=910645&view=rev Log: PIG-1234: Unable to create input slice for har:// files (pradeepkth)
Modified: hadoop/pig/branches/load-store-redesign/CHANGES.txt hadoop/pig/branches/load-store-redesign/src/org/apache/pig/LoadFunc.java hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLoad.java Modified: hadoop/pig/branches/load-store-redesign/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/CHANGES.txt?rev=910645&r1=910644&r2=910645&view=diff ============================================================================== --- hadoop/pig/branches/load-store-redesign/CHANGES.txt (original) +++ hadoop/pig/branches/load-store-redesign/CHANGES.txt Tue Feb 16 18:42:14 2010 @@ -22,6 +22,8 @@ INCOMPATIBLE CHANGES +PIG-1234: Unable to create input slice for har:// files (pradeepkth) + PIG-1200: Using TableInputFormat in HBaseStorage (zjffdu via pradeepkth) PIG-1148: Move splitable logic from pig latin to InputFormat (zjffdu via Modified: hadoop/pig/branches/load-store-redesign/src/org/apache/pig/LoadFunc.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/LoadFunc.java?rev=910645&r1=910644&r2=910645&view=diff ============================================================================== --- hadoop/pig/branches/load-store-redesign/src/org/apache/pig/LoadFunc.java (original) +++ hadoop/pig/branches/load-store-redesign/src/org/apache/pig/LoadFunc.java Tue Feb 16 18:42:14 2010 @@ -237,21 +237,20 @@ for (String fname: fnames) { Path p = new Path(fname.trim()); URI uri = p.toUri(); - String scheme = uri.getScheme(); - if (scheme != null) { - scheme = scheme.toLowerCase(); - } - - if (scheme != null && !scheme.equals(fsScheme)) { - throw new FrontendException("Incompatible file URI scheme: " - + scheme + " : " + fsScheme); - } - String path = uri.getPath(); // if the supplied location has an authority and is absolute, just // use it - if(uri.getAuthority() != null && p.isAbsolute()) { - fname = p.toString(); - } else { + if(uri.getAuthority() == null || ! p.isAbsolute()) { + String scheme = uri.getScheme(); + if (scheme != null) { + scheme = scheme.toLowerCase(); + } + + if (scheme != null && !scheme.equals(fsScheme)) { + throw new FrontendException("Incompatible file URI scheme: " + + scheme + " : " + fsScheme); + } + String path = uri.getPath(); + fname = (p.isAbsolute()) ? new Path(rootDir, path).toString() : new Path(curDir, path).toString(); Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLoad.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLoad.java?rev=910645&r1=910644&r2=910645&view=diff ============================================================================== --- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLoad.java (original) +++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLoad.java Tue Feb 16 18:42:14 2010 @@ -51,6 +51,7 @@ import org.apache.pig.impl.logicalLayer.LogicalPlanBuilder; import org.apache.pig.impl.plan.OperatorKey; import org.apache.pig.test.utils.GenPhyOp; +import org.apache.pig.test.utils.LogicalPlanTester; import org.apache.pig.test.utils.TestHelper; import org.junit.After; import org.junit.Before; @@ -63,6 +64,7 @@ static MiniCluster cluster = MiniCluster.buildCluster(); + @Override @Before public void setUp() throws Exception { FileLocalizer.deleteTempFiles(); @@ -72,6 +74,7 @@ }; } + @Override @After public void tearDown() throws Exception { } @@ -207,7 +210,16 @@ pc = pig.getPigContext(); checkLoadPath("usr/pig/{a,c},/usr/pig/b","/tmp/usr/pig/{a,c},/usr/pig/b"); } - } + } + + @Test + public void testNonDfsLocation() throws Exception { + LogicalPlanTester lpt = new LogicalPlanTester(); + String nonDfsUrl = "har://hdfs-namenode/user/foo/"; + LogicalPlan lp = lpt.buildPlan("a = load '" + nonDfsUrl + "';"); + LOLoad load = (LOLoad) lp.getRoots().get(0); + Assert.assertEquals(nonDfsUrl, load.getInputFile().getFileName()); + } private void checkLoadPath(String orig, String expected) throws Exception {