Author: daijy Date: Mon Dec 7 21:55:41 2009 New Revision: 888153 URL: http://svn.apache.org/viewvc?rev=888153&view=rev Log: PIG-1128: column pruning causing failure when foreach has user-specified schema
Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOForEach.java hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=888153&r1=888152&r2=888153&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Mon Dec 7 21:55:41 2009 @@ -45,8 +45,6 @@ OPTIMIZATIONS -PIG-922: Logical optimizer: push up project (daijy) - BUG FIXES PIG-1105: COUNT_STAR accumulate interface implementation cases failure @@ -85,6 +83,8 @@ INCOMPATIBLE CHANGES +PIG-922: Logical optimizer: push up project (daijy) + IMPROVEMENTS PIG-1084: Pig 0.6.0 Documentation improvements (chandec via olgan) @@ -182,6 +182,8 @@ OPTIMIZATIONS +PIG-922: Logical optimizer: push up project (daijy) + BUG FIXES PIG-1080: PigStorage may miss records when loading a file (rding via olgan) @@ -251,6 +253,9 @@ PIG-1060: MultiQuery optimization throws error for multi-level splits (rding via daijy) +PIG-1128: column pruning causing failure when foreach has user-specified +schema (daijy) + Release 0.5.0 INCOMPATIBLE CHANGES Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOForEach.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOForEach.java?rev=888153&r1=888152&r2=888153&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOForEach.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOForEach.java Mon Dec 7 21:55:41 2009 @@ -944,6 +944,12 @@ } while (planToRemove.size() > 0) { int index = planToRemove.get(planToRemove.size()-1); + if (mUserDefinedSchema!=null) { + for (int i=mUserDefinedSchema.size()-1;i>=0;i--) { + if (getRelevantPlan(0, i)==mForEachPlans.get(index)) + mUserDefinedSchema.remove(i); + } + } mForEachPlans.remove(index); mFlatten.remove(index); planToRemove.remove(planToRemove.size()-1); Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java?rev=888153&r1=888152&r2=888153&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java Mon Dec 7 21:55:41 2009 @@ -1453,5 +1453,28 @@ assertTrue(checkLogFileMessage(new String[]{"No column pruned for A", "No map keys pruned for A", "Columns pruned for B: $0", "No map keys pruned for B"})); - } + } + + // See PIG-1128 + @Test + public void testUserDefinedSchema() throws Exception { + pigServer.registerQuery("A = load '"+ Util.generateURI(tmpFile1.toString()) + "' AS ( c1 : chararray, c2 : int);"); + pigServer.registerQuery("B = foreach A generate c1 as c1 : chararray, c2 as c2 : int, 'CA' as state : chararray;"); + pigServer.registerQuery("C = foreach B generate c1 as c1 : chararray;"); + + Iterator<Tuple> iter = pigServer.openIterator("C"); + + assertTrue(iter.hasNext()); + Tuple t = iter.next(); + assertTrue(t.toString().equals("(1)")); + + assertTrue(iter.hasNext()); + t = iter.next(); + assertTrue(t.toString().equals("(2)")); + + assertFalse(iter.hasNext()); + + assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $1", + "No map keys pruned for A"})); + } }