Author: olga Date: Mon Sep 22 13:18:17 2008 New Revision: 697972 URL: http://svn.apache.org/viewvc?rev=697972&view=rev Log: PIG-430: Projections in nested filter and inside foreach do not work
Modified: incubator/pig/branches/types/CHANGES.txt incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld Modified: incubator/pig/branches/types/CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/CHANGES.txt?rev=697972&r1=697971&r2=697972&view=diff ============================================================================== --- incubator/pig/branches/types/CHANGES.txt (original) +++ incubator/pig/branches/types/CHANGES.txt Mon Sep 22 13:18:17 2008 @@ -227,3 +227,6 @@ PIG-411: Pig leaves HOD processes behind if Ctrl-C is used before HOD connection is fully established (olgan) + + PIG-430: Projections in nested filter and inside foreach do not work (sms + via olgan) Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=697972&r1=697971&r2=697972&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Mon Sep 22 13:18:17 2008 @@ -66,6 +66,7 @@ //a map of alias to logical operator for a quick lookup private Map<String, LogicalOperator> mapAliasOp; private static Log log = LogFactory.getLog(QueryParser.class); + private boolean bracketed = false; private long getNextId() { return nodeIdGen.getNextNodeId(scope); @@ -2084,11 +2085,12 @@ ( <FILTER> ( - LOOKAHEAD({ null != specs.get(getToken(1).image) }) t = <IDENTIFIER> {eOp = specs.get(t.image);} + LOOKAHEAD(NestedProject(over, specs, lp, input)) eOp = NestedProject(over, specs, lp, input) +| LOOKAHEAD({ null != specs.get(getToken(1).image) }) t = <IDENTIFIER> {eOp = specs.get(t.image);} | eOp = BaseEvalSpec(over, specs, lp, input) ) {subSchema = eOp.getSchema();} - <BY> cond = PCond(subSchema,null,conditionPlan,input) + <BY> cond = PCond(subSchema,null,conditionPlan,eOp) ) { lp.add(eOp); @@ -2122,7 +2124,8 @@ ( ( <ORDER> | <ARRANGE> ) ( - LOOKAHEAD({ null != specs.get(getToken(1).image) }) t = <IDENTIFIER> {eOp = specs.get(t.image);} + LOOKAHEAD(NestedProject(over, specs, lp, input)) eOp = NestedProject(over, specs, lp, input) +| LOOKAHEAD({ null != specs.get(getToken(1).image) }) t = <IDENTIFIER> {eOp = specs.get(t.image);} | eOp = BaseEvalSpec(over, specs, lp, input) ) <BY> @@ -2184,7 +2187,8 @@ ( <DISTINCT> ( - LOOKAHEAD({ null != specs.get(getToken(1).image) }) t = <IDENTIFIER> {eOp = specs.get(t.image);} + LOOKAHEAD(NestedProject(over, specs, lp, input)) eOp = NestedProject(over, specs, lp, input) +| LOOKAHEAD({ null != specs.get(getToken(1).image) }) t = <IDENTIFIER> {eOp = specs.get(t.image);} | eOp = BaseEvalSpec(over, specs, lp, input) ) ) @@ -2981,7 +2985,8 @@ int i; ExpressionOperator spec = null; log.trace("Entering BracketedSimpleProj"); - log.debug("eOp: " + eOp.getClass().getName()); + log.debug("eOp: " + eOp); + bracketed = true; } { ( @@ -2989,7 +2994,7 @@ | ("(" spec = SimpleProj(over,lp,eOp) ")") ) - {log.trace("Exiting BracketedSimpleProj");return spec;} + {log.trace("Exiting BracketedSimpleProj");bracketed=false; return spec;} } ExpressionOperator SimpleProj(Schema over, LogicalPlan lp, LogicalOperator eOp): @@ -3189,7 +3194,7 @@ log.debug("eOp: " + eOp.getClass().getName() + " " + eOp); lp.add(project); log.debug("DollarVar: Added operator " + project.getClass().getName() + " " + project + " to logical plan " + lp); - if(eOp instanceof ExpressionOperator) { + if((eOp instanceof ExpressionOperator) && (bracketed)) { lp.add(eOp); lp.connect(eOp, project); } @@ -3273,7 +3278,7 @@ try { lp.add(item); log.debug("AliasFieldOrSpec: Added operator " + item.getClass().getName() + " " + item + " to logical plan " + lp); - if(eOp instanceof ExpressionOperator) { + if((eOp instanceof ExpressionOperator) && (bracketed)) { lp.add(eOp); lp.connect(eOp, item); } Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=697972&r1=697971&r2=697972&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java Mon Sep 22 13:18:17 2008 @@ -26,6 +26,7 @@ import java.util.Properties; import java.net.URL; import java.util.List; +import java.util.ArrayList; import java.util.Set; import junit.framework.AssertionFailedError; @@ -1366,6 +1367,46 @@ assertTrue(Schema.equals(foreach.getSchema(), getSchemaFromString("name: bytearray, age: bytearray, gpa: bytearray, max_age: double"), false, true)); } + @Test + public void testQuery96() throws FrontendException, ParseException { + buildPlan("a = load 'input' as (name, age, gpa);"); + buildPlan("b = filter a by age < 20;"); + buildPlan("c = group b by age;"); + String query = "d = foreach c {" + + "cf = filter b by gpa < 3.0;" + + "cd = distinct cf.gpa;" + + "co = order cd by $0;" + + "generate group, flatten(co);" + + "};"; + LogicalPlan lp = buildPlan(query); + + LOForEach foreach = (LOForEach)lp.getLeaves().get(0); + ArrayList<LogicalPlan> foreachPlans = foreach.getForEachPlans(); + LogicalPlan flattenPlan = foreachPlans.get(1); + LogicalOperator project = flattenPlan.getLeaves().get(0); + assertTrue(project instanceof LOProject); + LogicalOperator sort = flattenPlan.getPredecessors(project).get(0); + assertTrue(sort instanceof LOSort); + LogicalOperator distinct = flattenPlan.getPredecessors(sort).get(0); + assertTrue(distinct instanceof LODistinct); + + //testing the presence of the nested foreach + LogicalOperator nestedForeach = flattenPlan.getPredecessors(distinct).get(0); + assertTrue(nestedForeach instanceof LOForEach); + LogicalPlan nestedForeachPlan = ((LOForEach)nestedForeach).getForEachPlans().get(0); + LogicalOperator nestedProject = nestedForeachPlan.getRoots().get(0); + assertTrue(nestedProject instanceof LOProject); + assertTrue(((LOProject)nestedProject).getCol() == 2); + + //testing the filter inner plan for the absence of the project connected to project + LogicalOperator filter = flattenPlan.getPredecessors(nestedForeach).get(0); + assertTrue(filter instanceof LOFilter); + LogicalPlan comparisonPlan = ((LOFilter)filter).getComparisonPlan(); + LOLesserThan lessThan = (LOLesserThan)comparisonPlan.getLeaves().get(0); + LOProject filterProject = (LOProject)lessThan.getLhsOperand(); + assertTrue(null == comparisonPlan.getPredecessors(filterProject)); + } + private Schema getSchemaFromString(String schemaString) throws ParseException { return getSchemaFromString(schemaString, DataType.BYTEARRAY); } Modified: incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld?rev=697972&r1=697971&r2=697972&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld Mon Sep 22 13:18:17 2008 @@ -6,9 +6,9 @@ | | | | | Equal To[boolean] - Test-Plan-Builder-14 | | | -| | |---Project[bytearray][1] - Test-Plan-Builder-12 +| | |---Project[Unknown][1] - Test-Plan-Builder-12 | | | -| | |---Project[bytearray][2] - Test-Plan-Builder-13 +| | |---Project[Unknown][2] - Test-Plan-Builder-13 | | | |---Project[tuple][0] - Test-Plan-Builder-10 |