Author: daijy
Date: Mon Mar 15 03:24:05 2010
New Revision: 923042
URL: http://svn.apache.org/viewvc?rev=923042&view=rev
Log:
PIG-1272: Column pruner causes wrong results
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java
hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQuery.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=923042&r1=923041&r2=923042&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Mon Mar 15 03:24:05 2010
@@ -150,6 +150,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-1272: Column pruner causes wrong results (daijy)
+
PIG-1275: empty bag in PigStorage read as null (daijy)
PIG-1252: Diamond splitter does not generate correct results when using
Modified:
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java?rev=923042&r1=923041&r2=923042&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java
Mon Mar 15 03:24:05 2010
@@ -220,12 +220,25 @@ public class PruneColumns extends Logica
{
List<RequiredFields> requiredInputFieldsList = new
ArrayList<RequiredFields>();
RequiredFields requiredFields = new RequiredFields(false);
- for (RequiredFields rf : requiredOutputInfo.requiredFieldsList)
+ for (int i=0;i<mPlan.getSuccessors(rlo).size();i++)
{
+ RequiredFields rf = null;
+ try {
+ rf = requiredOutputInfo.requiredFieldsList.get(i);
+ } catch (Exception e) {
+ }
if (rf!=null)
{
rf.reIndex(0);
requiredFields.merge(rf);
+ } else {
+ // need all fields
+ List<Pair<Integer, Integer>> l = new
ArrayList<Pair<Integer, Integer>>();
+ for (int j=0;j<rlo.getSchema().size();j++)
+ l.add(new Pair<Integer, Integer>(0, j));
+ rf = new RequiredFields(l);
+ requiredFields.merge(rf);
+ break;
}
}
requiredInputFieldsList.add(requiredFields);
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQuery.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQuery.java?rev=923042&r1=923041&r2=923042&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQuery.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQuery.java Mon Mar 15
03:24:05 2010
@@ -322,9 +322,9 @@ public class TestMultiQuery {
List<Tuple> expectedResults =
Util.getTuplesFromConstantTupleStrings(
new String[] {
- "(1L,1L,'apple',1L,{(1L)})",
- "(2L,2L,'orange',2L,{(2L)})",
- "(3L,3L,'persimmon',3L,{(3L)})"
+ "(1L,'apple',3,1L,'apple',1L,{(1L)})",
+ "(2L,'orange',4,2L,'orange',2L,{(2L)})",
+ "(3L,'persimmon',5,3L,'persimmon',3L,{(3L)})"
});
int counter = 0;
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java?rev=923042&r1=923041&r2=923042&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java Mon Mar 15
03:24:05 2010
@@ -1844,5 +1844,25 @@ public class TestPruneColumn extends Tes
assertTrue(checkLogFileMessage(new String[]{"No column pruned for A",
"No map keys pruned for A", "[0,1,2]"}));
}
+
+ // See PIG-1272
+ @Test
+ public void testSplit4() throws Exception {
+ pigServer.registerQuery("A = load '"+
Util.generateURI(tmpFile1.toString(), pigServer.getPigContext()) + "' AS (a0,
a1, a2);");
+ pigServer.registerQuery("B = foreach A generate a0;");
+ pigServer.registerQuery("C = join A by a0, B by a0;");
+ Iterator<Tuple> iter = pigServer.openIterator("C");
+
+ assertTrue(iter.hasNext());
+ Tuple t = iter.next();
+ assertTrue(t.toString().equals("(1,2,3,1)"));
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+ assertTrue(t.toString().equals("(2,5,2,2)"));
+
+ assertTrue(checkLogFileMessage(new String[]{"No column pruned for A",
+ "No map keys pruned for A"}));
+ }
}
