Author: sms
Date: Mon Jul 13 23:36:40 2009
New Revision: 793750
URL: http://svn.apache.org/viewvc?rev=793750&view=rev
Log:
PIG-877: Push up filter does not account for added columns in foreach
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=793750&r1=793749&r2=793750&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Mon Jul 13 23:36:40 2009
@@ -40,6 +40,9 @@
BUG FIXES
+ PIG-877: Push up filter does not account for added columns in foreach
+ (sms)
+
PIG-883: udf import list does not send to the backend (daijy)
PIG-881: Pig should ship load udfs to the backend (daijy)
Modified:
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java?rev=793750&r1=793749&r2=793750&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PushUpFilter.java
Mon Jul 13 23:36:40 2009
@@ -402,6 +402,10 @@
for (Integer input : predInputs) {
List<ProjectionMap.Column> inputList =
(List<ProjectionMap.Column>) mappedFields
.get(input);
+ // inputList is null -> the required field is added
+ if(inputList == null) {
+ return new Pair<Boolean, Set<Integer>>(false, null);
+ }
for (ProjectionMap.Column column : inputList) {
// TODO
// Check if the column has a cast
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java?rev=793750&r1=793749&r2=793750&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPushUpFilter.java Mon Jul 13
23:36:40 2009
@@ -276,6 +276,24 @@
assertEquals(filter, lp.getPredecessors(foreach).get(0));
}
+
+ @Test
+ public void testFilterForeachAddedField() throws Exception {
+ planTester.buildPlan("A = load 'myfile' as (name, age, gpa);");
+ planTester.buildPlan("B = foreach A generate $1, $2, COUNT({(1)});");
+ LogicalPlan lp = planTester.buildPlan("C = filter B by $2 < 18;");
+
+ planTester.setPlan(lp);
+ planTester.setProjectionMap(lp);
+
+ PushUpFilter pushUpFilter = new PushUpFilter(lp);
+
+ assertTrue(!pushUpFilter.check(lp.getLeaves()));
+ assertTrue(pushUpFilter.getSwap() == false);
+ assertTrue(pushUpFilter.getPushBefore() == false);
+ assertTrue(pushUpFilter.getPushBeforeInput() == -1);
+
+ }
@Test
public void testFilterForeachCast() throws Exception {
