Author: daijy Date: Fri Mar 19 06:47:53 2010 New Revision: 925105 URL: http://svn.apache.org/viewvc?rev=925105&view=rev Log: PIG-1289: PIG Join fails while doing a filter on joined data
Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=925105&r1=925104&r2=925105&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Fri Mar 19 06:47:53 2010 @@ -157,6 +157,8 @@ OPTIMIZATIONS BUG FIXES +PIG-1289: PIG Join fails while doing a filter on joined data (daijy) + PIG-1266: Show spill count on the pig console at the end of the job (sriranjan via rding) Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java?rev=925105&r1=925104&r2=925105&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java Fri Mar 19 06:47:53 2010 @@ -31,6 +31,7 @@ import org.apache.pig.impl.logicalLayer. import org.apache.pig.impl.logicalLayer.LOCast; import org.apache.pig.impl.logicalLayer.LOCogroup; import org.apache.pig.impl.logicalLayer.LOCross; +import org.apache.pig.impl.logicalLayer.LOIsNull; import org.apache.pig.impl.logicalLayer.LOJoin; import org.apache.pig.impl.logicalLayer.LOFilter; import org.apache.pig.impl.logicalLayer.LOForEach; @@ -231,8 +232,32 @@ public class PushUpFilter extends Logica return false; } } - mPushBefore = true; + mPushBeforeInput = grandParentIndexes.iterator().next(); + + if (predecessor instanceof LOJoin) { + boolean otherBranchContainOuter = false; + boolean sawInner = false; + for (int i=0;i<=mPlan.getSuccessors(predecessor).size();i++) { + // We do not push filter if any other branch is outer + // See PIG-1289 + // Also in LOJoin, innerFlag==true indicate that branch is the outer join side + // which has the exact opposite semantics + // If all innerFlag is true, that implies a regular join + if (i!=mPushBeforeInput && ((LOJoin)predecessor).getInnerFlags()[i]) { + otherBranchContainOuter = true; + } + if (((LOJoin)predecessor).getInnerFlags()[i]==false) { + sawInner = true; + } + } + if (otherBranchContainOuter && sawInner) { + mPushBeforeInput = -1; + return false; + } + } + + mPushBefore = true; return true; } else if (predecessor instanceof LOForEach) { Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java?rev=925105&r1=925104&r2=925105&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java Fri Mar 19 06:47:53 2010 @@ -1068,6 +1068,24 @@ public class TestPushUpFilter extends ju assertTrue(pushUpFilter.getPushBefore() == false); assertTrue(pushUpFilter.getPushBeforeInput() == -1); } + + @Test + public void testOutJoin() throws Exception { + planTester.buildPlan("A = load 'myfile' as (name, age, gpa);"); + planTester.buildPlan("B = load 'anotherfile' as (name);"); + planTester.buildPlan("C = join A by name LEFT OUTER, B by name;"); + LogicalPlan lp = planTester.buildPlan("D = filter C by B::name is null;"); + + planTester.setPlan(lp); + planTester.setProjectionMap(lp); + + PushUpFilter pushUpFilter = new PushUpFilter(lp); + + assertTrue(!pushUpFilter.check(lp.getLeaves())); + assertTrue(pushUpFilter.getSwap() == false); + assertTrue(pushUpFilter.getPushBefore() == false); + assertTrue(pushUpFilter.getPushBeforeInput() == -1); + } }