Author: daijy
Date: Fri Jul 9 18:32:16 2010
New Revision: 962626
URL: http://svn.apache.org/viewvc?rev=962626&view=rev
Log:
PIG-1484: BinStorage should support comma seperated path
Modified:
hadoop/pig/branches/branch-0.7/CHANGES.txt
hadoop/pig/branches/branch-0.7/src/org/apache/pig/builtin/BinStorage.java
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestEvalPipeline2.java
Modified: hadoop/pig/branches/branch-0.7/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/CHANGES.txt?rev=962626&r1=962625&r2=962626&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.7/CHANGES.txt Fri Jul 9 18:32:16 2010
@@ -196,6 +196,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-1484: BinStorage should support comma seperated path (daijy)
+
PIG-1467: order by fail when set "fs.file.impl.disable.cache" to true (daijy)
PIG-1443: DefaultTuple underestimate the memory footprint for string (daijy)
Modified:
hadoop/pig/branches/branch-0.7/src/org/apache/pig/builtin/BinStorage.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/src/org/apache/pig/builtin/BinStorage.java?rev=962626&r1=962625&r2=962626&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/src/org/apache/pig/builtin/BinStorage.java
(original)
+++ hadoop/pig/branches/branch-0.7/src/org/apache/pig/builtin/BinStorage.java
Fri Jul 9 18:32:16 2010
@@ -395,12 +395,17 @@ implements LoadCaster, StoreFuncInterfac
// we can treat either local or hadoop mode as hadoop mode - hence
// we can use HDataStorage and FileLocalizer.openDFSFile below
HDataStorage storage = new HDataStorage(props);
- if (!FileLocalizer.fileExists(location, storage)) {
- // At compile time in batch mode, the file may not exist
- // (such as intermediate file). Just return null - the
- // same way as we would if we did not get a valid record
- return null;
+
+ // At compile time in batch mode, the file may not exist
+ // (such as intermediate file). Just return null - the
+ // same way as we would if we did not get a valid record
+ String[] locations = getPathStrings(location);
+ for (String loc : locations) {
+ if (!FileLocalizer.fileExists(loc, storage)) {
+ return null;
+ }
}
+
ReadToEndLoader loader = new ReadToEndLoader(this, conf, location, 0);
// get the first record from the input file
// and figure out the schema from the data in
Modified:
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestEvalPipeline2.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestEvalPipeline2.java?rev=962626&r1=962625&r2=962626&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestEvalPipeline2.java
(original)
+++
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestEvalPipeline2.java
Fri Jul 9 18:32:16 2010
@@ -490,4 +490,46 @@ public class TestEvalPipeline2 extends T
Util.deleteFile(cluster, "table_testNestedDescSort");
}
+ // See PIG-1484
+ @Test
+ public void testBinStorageCommaSeperatedPath() throws Exception{
+ String[] input = {
+ "1\t3",
+ "2\t4",
+ "3\t5"
+ };
+
+ Util.createInputFile(cluster, "table_simple1", input);
+ pigServer.setBatchOn();
+ pigServer.registerQuery("A = LOAD 'table_simple1' as (a0, a1);");
+ pigServer.registerQuery("store A into 'table_simple1.bin' using
BinStorage();");
+ pigServer.registerQuery("store A into 'table_simple2.bin' using
BinStorage();");
+
+ pigServer.executeBatch();
+
+ pigServer.registerQuery("A = LOAD
'table_simple1.bin,table_simple2.bin' using BinStorage();");
+ Iterator<Tuple> iter = pigServer.openIterator("A");
+
+ Tuple t = iter.next();
+ assertTrue(t.toString().equals("(1,3)"));
+
+ t = iter.next();
+ assertTrue(t.toString().equals("(2,4)"));
+
+ t = iter.next();
+ assertTrue(t.toString().equals("(3,5)"));
+
+ t = iter.next();
+ assertTrue(t.toString().equals("(1,3)"));
+
+ t = iter.next();
+ assertTrue(t.toString().equals("(2,4)"));
+
+ t = iter.next();
+ assertTrue(t.toString().equals("(3,5)"));
+
+ assertFalse(iter.hasNext());
+ }
+
+
}
