Author: pradeepkth
Date: Wed May 13 22:58:53 2009
New Revision: 774582

URL: http://svn.apache.org/viewvc?rev=774582&view=rev
Log:
PIG-804: problem with lineage with double map redirection (pradeepkth)

Modified:
    hadoop/pig/trunk/CHANGES.txt
    hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
    hadoop/pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidator.java

Modified: hadoop/pig/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=774582&r1=774581&r2=774582&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Wed May 13 22:58:53 2009
@@ -44,6 +44,8 @@
 
 BUG FIXES
 
+PIG-804: problem with lineage with double map redirection (pradeepkth)
+
 PIG-733: Order by sampling dumps entire sample to hdfs which causes dfs
 "FileSystem closed" error on large input (pradeepkth)
 

Modified: 
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=774582&r1=774581&r2=774582&view=diff
==============================================================================
--- 
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt 
(original)
+++ 
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt 
Wed May 13 22:58:53 2009
@@ -2646,6 +2646,13 @@
         if(castRequired) {
             cast = new LOCast(lp, new OperatorKey(scope, getNextId()), 
fs.type);
             fs.alias = exprOp.getFieldSchema().alias;
+            // unset the field schema computed above since it should
+            // be recomputed later from the TypeCheckingVisitor
+            // This is because operators might be added/removed
+            // from the plan which might affect the field schema. So
+            // the TypeCheckingVisitor would be right place to
+            // compute the field schema
+            exprOp.unsetFieldSchema();
             cast.setFieldSchema(fs);
             lp.add(cast);
                    log.debug("Added operator " + cast.getClass().getName() + " 
" + cast + " to logical plan " + lp);

Modified: 
hadoop/pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidator.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidator.java?rev=774582&r1=774581&r2=774582&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidator.java 
(original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidator.java 
Wed May 13 22:58:53 2009
@@ -31,8 +31,10 @@
 import org.apache.pig.impl.logicalLayer.* ;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema ;
+import org.apache.pig.impl.plan.DepthFirstWalker;
 import org.apache.pig.impl.plan.PlanValidationException;
 import org.apache.pig.impl.plan.CompilationMessageCollector;
+import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.util.MultiMap;
 import org.apache.pig.data.*;
 import org.apache.pig.impl.io.FileSpec;
@@ -5534,6 +5536,52 @@
     }
 
     @Test
+    public void testMapLookupLineage3() throws Throwable {
+        planTester.buildPlan("a = load 'a' as (s, m, l);") ;
+        planTester.buildPlan("b = foreach a generate s#'src_spaceid' AS 
vspaceid, flatten(l#'viewinfo') as viewinfo ;") ;
+        LogicalPlan plan = planTester.buildPlan("c = foreach b generate 
(chararray)vspaceid#'foo', (chararray)viewinfo#'pos' as position;") ;
+
+        // validate
+        CompilationMessageCollector collector = new 
CompilationMessageCollector() ;
+        TypeCheckingValidator typeValidator = new TypeCheckingValidator() ;
+        typeValidator.validate(plan, collector) ;
+
+        printMessageCollector(collector) ;
+        printTypeGraph(plan) ;
+        planTester.printPlan(plan, 
TypeCheckingTestUtil.getCurrentMethodName());
+
+        if (collector.hasError()) {
+            throw new AssertionError("Expect no  error") ;
+        }
+
+        CastFinder cf = new CastFinder(plan);
+        cf.visit();
+        List<LOCast> casts = cf.casts;
+        for (LOCast cast : casts) {
+            
assertTrue(cast.getLoadFuncSpec().getClassName().startsWith("org.apache.pig.builtin.PigStorage"));
    
+        }
+    }
+    
+    class CastFinder extends LOVisitor {
+        List<LOCast> casts = new ArrayList<LOCast>();
+        /**
+         * 
+         */
+        public CastFinder(LogicalPlan lp) {
+            // TODO Auto-generated constructor stub
+            super(lp, new DepthFirstWalker<LogicalOperator, LogicalPlan>(lp));
+        }
+        
+        /* (non-Javadoc)
+         * @see 
org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOCast)
+         */
+        @Override
+        protected void visit(LOCast cast) throws VisitorException {
+            casts.add(cast);
+        }
+    }
+    
+    @Test
     public void testBincond() throws Throwable {
         planTester.buildPlan("a = load 'a' as (name: chararray, age: int, gpa: 
float);") ;
         planTester.buildPlan("b = group a by name;") ;


Reply via email to