Author: pisong
Date: Wed Jul 16 05:21:42 2008
New Revision: 677255

URL: http://svn.apache.org/viewvc?rev=677255&view=rev
Log:
PIG-315 "AS" in foreach problem and flattened member aggregate type check

Modified:
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
    incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java
    incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=677255&r1=677254&r2=677255&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
 (original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
 Wed Jul 16 05:21:42 2008
@@ -1570,10 +1570,17 @@
 }
 {
        (
-       item = FlattenedGenerateItem(over, specs, generatePlan = new 
LogicalPlan(), input, flattenList) {generatePlans.add(generatePlan);}
-       ("," item = FlattenedGenerateItem(over, specs, generatePlan = new 
LogicalPlan(), input, flattenList) {generatePlans.add(generatePlan);})*
-       
-       )
+        item = FlattenedGenerateItem(over, specs, generatePlan = new 
LogicalPlan(), input, flattenList)
+            {
+                generatePlans.add(generatePlan);
+            }
+        ("," item = FlattenedGenerateItem(over, specs, generatePlan = new 
LogicalPlan(), input, flattenList)
+            {
+                generatePlans.add(generatePlan);
+            }
+        )*
+
+    )
        {
                LogicalOperator generate = new LOGenerate(lp, new 
OperatorKey(scope, getNextId()), generatePlans, flattenList);
                lp.add(generate);
@@ -1617,13 +1624,32 @@
                }
        )
        )
-       [ <AS> (LOOKAHEAD(2) "(" schema = TupleSchema() ")" {if(schema.size() > 
1) {throw new ParseException("Schema mismatch");} fs = 
schema.getFields().get(0);} | fs = AtomSchema() )]
+        [ <AS> (LOOKAHEAD(2) "(" schema = TupleSchema() ")"
+
+          { if (schema.size() > 1) {
+                throw new ParseException("Schema mismatch");
+            }
+
+            fs = schema.getFields().get(0);
+          }
+          | fs = AtomSchema() )
+        ]
        )
        {
                log.debug("item: " + item.getClass().getName());
-               if(null != fs) {
-                       item.setFieldSchema(fs); 
-               }
+        if(null != fs) {
+            // We're only interested in alias here
+            // target type should be preserved
+            Schema.FieldSchema tmpFs = item.getFieldSchema() ;
+            if (tmpFs != null) {
+                tmpFs.alias = fs.alias ;
+                item.setFieldSchema(tmpFs);    
+            }
+            else {
+                tmpFs = new Schema.FieldSchema(fs.alias, DataType.BYTEARRAY) ;
+                item.setFieldSchema(tmpFs);
+            }
+        }
                flattenList.add(flatten);
                log.trace("Exiting FlattenedGenerateItem");
                return item;

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java?rev=677255&r1=677254&r2=677255&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
 (original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
 Wed Jul 16 05:21:42 2008
@@ -27,6 +27,7 @@
 
 import org.apache.pig.EvalFunc;
 import org.apache.pig.FuncSpec;
+import org.apache.pig.Algebraic;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.logicalLayer.LOConst;
@@ -1314,9 +1315,34 @@
             }
             
         }
+
+        EvalFunc<?> ef = (EvalFunc<?>) 
PigContext.instantiateFuncFromSpec(func.getFuncSpec());
+
+        // If the function is algebraic and the project is just sentinel
+        // (special case when we apply aggregate on flattened members)
+        // then it will never match algebraic functions' schemas
+        // without this
+
+        // Assuming all aggregates has only one argument at this stage
+        ExpressionOperator tmpExp = func.getArguments().get(0) ;
+        if ( (ef instanceof Algebraic)
+             && (tmpExp instanceof LOProject)
+             && (((LOProject)tmpExp).getSentinel())) {
+
+            FieldSchema tmpField ;
+
+            try {
+                // embed the schema above inside a bag
+                tmpField = new FieldSchema(null, s, DataType.BAG) ;
+            }
+            catch (FrontendException e) {
+                throw new VisitorException(e) ;
+            }
+
+            s = new Schema(tmpField) ;
+        }
         
         // ask the EvalFunc what types of inputs it can handle
-        EvalFunc<?> ef = (EvalFunc<?>) 
PigContext.instantiateFuncFromSpec(func.getFuncSpec());
         List<FuncSpec> funcSpecs = null;
         try {
             funcSpecs = ef.getArgToFuncMapping();    
@@ -1337,7 +1363,7 @@
             if(matchingSpec == null) {
                 StringBuilder sb = new StringBuilder();
                 sb.append(func.getFuncSpec());
-                sb.append("does not work with inputs of type ");
+                sb.append(" does not work with inputs of type ");
                 sb.append(s);
                 throw new VisitorException(sb.toString());
             } else {
@@ -2171,8 +2197,10 @@
 
         }
         catch (FrontendException pe) {
-            String msg = "Problem resolving LOForEach schema" ;
+            String msg = "Problem resolving LOForEach schema " 
+                         + pe.getMessage() ;
             msgCollector.collect(msg, MessageType.Error) ;
+            log.debug(pe);
             VisitorException vse = new VisitorException(msg) ;
             throw vse ;
         }

Modified: 
incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java?rev=677255&r1=677254&r2=677255&view=diff
==============================================================================
--- 
incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java 
(original)
+++ 
incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java 
Wed Jul 16 05:21:42 2008
@@ -48,8 +48,6 @@
             }
         }
         ps.close();
-        String query = "myid = foreach (group (load 'file:" + tmpFile + "') 
all) generate group, COUNT($1) ;";
-        System.out.println(query);
         pig.registerQuery(" a = group (load 'file:" + tmpFile + "') by 
($0,$1);");
         pig.registerQuery("b = foreach a generate flatten(group), 
SUM($1.$2);");
         Iterator<Tuple> it = pig.openIterator("b");

Modified: 
incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java?rev=677255&r1=677254&r2=677255&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java 
(original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java 
Wed Jul 16 05:21:42 2008
@@ -242,6 +242,21 @@
 
     */
 
+    public void testSUM1() throws Throwable {
+        TypeCheckingTestUtil.printCurrentMethodName() ;
+        planTester.buildPlan("a = load ':INPATH:/singlefile/studenttab10k' as 
(name:chararray, age:int, gpa:double);") ;
+        LogicalPlan plan1 = planTester.buildPlan("b = foreach a generate 
(long)age as age, (int)gpa as gpa;") ;
+        LogicalPlan plan2 = planTester.buildPlan("c = foreach b generate 
SUM(age), SUM(gpa);") ;
+        planTester.typeCheckPlan(plan2);
+    }
+
+    public void testSUM2() throws Throwable {
+        TypeCheckingTestUtil.printCurrentMethodName() ;
+        planTester.buildPlan("a = group (load 'file:\" + tmpFile + \"') by 
($0,$1);") ;
+        LogicalPlan plan1 = planTester.buildPlan("b = foreach a generate 
flatten(group), SUM($1.$2);") ;
+        planTester.typeCheckPlan(plan1);
+    }
+
 
 
 }


Reply via email to