Author: pradeepkth Date: Wed May 13 22:59:51 2009 New Revision: 774583 URL: http://svn.apache.org/viewvc?rev=774583&view=rev Log: PIG-804: problem with lineage with double map redirection (pradeepkth)
Modified: hadoop/pig/branches/pre-multiquery-phase2/CHANGES.txt hadoop/pig/branches/pre-multiquery-phase2/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt hadoop/pig/branches/pre-multiquery-phase2/test/org/apache/pig/test/TestTypeCheckingValidator.java Modified: hadoop/pig/branches/pre-multiquery-phase2/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/branches/pre-multiquery-phase2/CHANGES.txt?rev=774583&r1=774582&r2=774583&view=diff ============================================================================== --- hadoop/pig/branches/pre-multiquery-phase2/CHANGES.txt (original) +++ hadoop/pig/branches/pre-multiquery-phase2/CHANGES.txt Wed May 13 22:59:51 2009 @@ -32,6 +32,8 @@ BUG FIXES +PIG-804: problem with lineage with double map redirection (pradeepkth) + PIG-733: Order by sampling dumps entire sample to hdfs which causes dfs "FileSystem closed" error on large input (pradeepkth) Modified: hadoop/pig/branches/pre-multiquery-phase2/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt URL: http://svn.apache.org/viewvc/hadoop/pig/branches/pre-multiquery-phase2/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=774583&r1=774582&r2=774583&view=diff ============================================================================== --- hadoop/pig/branches/pre-multiquery-phase2/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt (original) +++ hadoop/pig/branches/pre-multiquery-phase2/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Wed May 13 22:59:51 2009 @@ -2502,6 +2502,13 @@ if(castRequired) { cast = new LOCast(lp, new OperatorKey(scope, getNextId()), fs.type); fs.alias = exprOp.getFieldSchema().alias; + // unset the field schema computed above since it should + // be recomputed later from the TypeCheckingVisitor + // This is because operators might be added/removed + // from the plan which might affect the field schema. So + // the TypeCheckingVisitor would be right place to + // compute the field schema + exprOp.unsetFieldSchema(); cast.setFieldSchema(fs); lp.add(cast); log.debug("Added operator " + cast.getClass().getName() + " " + cast + " to logical plan " + lp); Modified: hadoop/pig/branches/pre-multiquery-phase2/test/org/apache/pig/test/TestTypeCheckingValidator.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/pre-multiquery-phase2/test/org/apache/pig/test/TestTypeCheckingValidator.java?rev=774583&r1=774582&r2=774583&view=diff ============================================================================== --- hadoop/pig/branches/pre-multiquery-phase2/test/org/apache/pig/test/TestTypeCheckingValidator.java (original) +++ hadoop/pig/branches/pre-multiquery-phase2/test/org/apache/pig/test/TestTypeCheckingValidator.java Wed May 13 22:59:51 2009 @@ -31,8 +31,10 @@ import org.apache.pig.impl.logicalLayer.* ; import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema ; +import org.apache.pig.impl.plan.DepthFirstWalker; import org.apache.pig.impl.plan.PlanValidationException; import org.apache.pig.impl.plan.CompilationMessageCollector; +import org.apache.pig.impl.plan.VisitorException; import org.apache.pig.impl.util.MultiMap; import org.apache.pig.data.*; import org.apache.pig.impl.io.FileSpec; @@ -5534,6 +5536,52 @@ } @Test + public void testMapLookupLineage3() throws Throwable { + planTester.buildPlan("a = load 'a' as (s, m, l);") ; + planTester.buildPlan("b = foreach a generate s#'src_spaceid' AS vspaceid, flatten(l#'viewinfo') as viewinfo ;") ; + LogicalPlan plan = planTester.buildPlan("c = foreach b generate (chararray)vspaceid#'foo', (chararray)viewinfo#'pos' as position;") ; + + // validate + CompilationMessageCollector collector = new CompilationMessageCollector() ; + TypeCheckingValidator typeValidator = new TypeCheckingValidator() ; + typeValidator.validate(plan, collector) ; + + printMessageCollector(collector) ; + printTypeGraph(plan) ; + planTester.printPlan(plan, TypeCheckingTestUtil.getCurrentMethodName()); + + if (collector.hasError()) { + throw new AssertionError("Expect no error") ; + } + + CastFinder cf = new CastFinder(plan); + cf.visit(); + List<LOCast> casts = cf.casts; + for (LOCast cast : casts) { + assertTrue(cast.getLoadFuncSpec().getClassName().startsWith("org.apache.pig.builtin.PigStorage")); + } + } + + class CastFinder extends LOVisitor { + List<LOCast> casts = new ArrayList<LOCast>(); + /** + * + */ + public CastFinder(LogicalPlan lp) { + // TODO Auto-generated constructor stub + super(lp, new DepthFirstWalker<LogicalOperator, LogicalPlan>(lp)); + } + + /* (non-Javadoc) + * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOCast) + */ + @Override + protected void visit(LOCast cast) throws VisitorException { + casts.add(cast); + } + } + + @Test public void testBincond() throws Throwable { planTester.buildPlan("a = load 'a' as (name: chararray, age: int, gpa: float);") ; planTester.buildPlan("b = group a by name;") ;