Author: daijy Date: Thu Jul 22 01:13:13 2010 New Revision: 966471 URL: http://svn.apache.org/viewvc?rev=966471&view=rev Log: PIG-1507: Full outer join fails while doing a filter on joined data
Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=966471&r1=966470&r2=966471&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Thu Jul 22 01:13:13 2010 @@ -347,6 +347,8 @@ OPTIMIZATIONS BUG FIXES +PIG-1507: Full outer join fails while doing a filter on joined data (daijy) + PIG-1493: Column Pruner throw exception "inconsistent pruning" (daijy) PIG-1484: BinStorage should support comma seperated path (daijy) Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java?rev=966471&r1=966470&r2=966471&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java Thu Jul 22 01:13:13 2010 @@ -244,6 +244,7 @@ public class PushUpFilter extends Logica // Also in LOJoin, innerFlag==true indicate that branch is the outer join side // which has the exact opposite semantics // If all innerFlag is true, that implies a regular join + // If all innerFlag is false, means a outer join, in this case, we can not push up filter for any path (See PIG-1507) if (i!=mPushBeforeInput && ((LOJoin)predecessor).getInnerFlags()[i]) { otherBranchContainOuter = true; } @@ -251,7 +252,13 @@ public class PushUpFilter extends Logica sawInner = true; } } - if (otherBranchContainOuter && sawInner) { + if (!otherBranchContainOuter && ((LOJoin)predecessor).getInnerFlags()[mPushBeforeInput]==false) // all innerFlag is false, implies an outer join + { + mPushBeforeInput = -1; + return false; + } + if (otherBranchContainOuter && sawInner) // If it is not a regular join and the path we push is on inner side + { mPushBeforeInput = -1; return false; } Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java?rev=966471&r1=966470&r2=966471&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java Thu Jul 22 01:13:13 2010 @@ -1074,6 +1074,7 @@ public class TestPushUpFilter extends ju assertTrue(pushUpFilter.getPushBeforeInput() == -1); } + // See PIG-1289 @Test public void testOutJoin() throws Exception { planTester.buildPlan("A = load 'myfile' as (name, age, gpa);"); @@ -1091,6 +1092,25 @@ public class TestPushUpFilter extends ju assertTrue(pushUpFilter.getPushBefore() == false); assertTrue(pushUpFilter.getPushBeforeInput() == -1); } + + // See PIG-1507 + @Test + public void testFullOutJoin() throws Exception { + planTester.buildPlan("A = load 'myfile' as (d1:int);"); + planTester.buildPlan("B = load 'anotherfile' as (d2:int);"); + planTester.buildPlan("c = join A by d1 full outer, B by d2;"); + LogicalPlan lp = planTester.buildPlan("d = filter c by d2 is null;"); + + planTester.setPlan(lp); + planTester.setProjectionMap(lp); + + PushUpFilter pushUpFilter = new PushUpFilter(lp); + + assertTrue(!pushUpFilter.check(lp.getLeaves())); + assertTrue(pushUpFilter.getSwap() == false); + assertTrue(pushUpFilter.getPushBefore() == false); + assertTrue(pushUpFilter.getPushBeforeInput() == -1); + } }