Author: gates
Date: Tue May 20 14:41:07 2008
New Revision: 658450

URL: http://svn.apache.org/viewvc?rev=658450&view=rev
Log:
PIG-159 Checked in Santhosh's changes to fix a few bugs in LOCross, LOUnion, 
change INTEGER keyword to INT.


Modified:
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
    
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
    
incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java 
(original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java 
Tue May 20 14:41:07 2008
@@ -64,13 +64,12 @@
     @Override
     public Schema getSchema() throws FrontendException {
         if (!mIsSchemaComputed && (null == mSchema)) {
-            Collection<LogicalOperator> pred = mPlan.getPredecessors(this);
             List<Schema.FieldSchema> fss = new ArrayList<Schema.FieldSchema>();
             Map<Schema.FieldSchema, String> flattenAlias = new 
HashMap<Schema.FieldSchema, String>();
             Map<String, Boolean> inverseFlattenAlias = new HashMap<String, 
Boolean>();
             Map<String, Integer> aliases = new HashMap<String, Integer>();
 
-            for (LogicalOperator op : pred) {
+            for (LogicalOperator op : mInputs) {
                 String opAlias = op.getAlias();
                 Schema s = op.getSchema();
 

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java 
(original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java 
Tue May 20 14:41:07 2008
@@ -33,7 +33,7 @@
 public class LOSplit extends LogicalOperator {
     private static final long serialVersionUID = 2L;
 
-    private Map<String, LogicalPlan> mCondPlans;
+    //private Map<String, LogicalPlan> mCondPlans;
     private ArrayList<LogicalOperator> mOutputs;
     private static Log log = LogFactory.getLog(LOSplit.class);
 
@@ -48,17 +48,17 @@
      *            list of conditions for the split
      */
     public LOSplit(LogicalPlan plan, OperatorKey key,
-            ArrayList<LogicalOperator> outputs,
-            Map<String, LogicalPlan> condPlans) {
+            ArrayList<LogicalOperator> outputs) {
+           // Map<String, LogicalPlan> condPlans) {
         super(plan, key);
         mOutputs = outputs;
-        mCondPlans = condPlans;
+        //mCondPlans = condPlans;
     }
 
     public List<LogicalOperator> getOutputs() {
         return mOutputs;
     }
-
+/*
     public Collection<LogicalPlan> getConditionPlans() {
         return mCondPlans.values();
     }
@@ -70,7 +70,7 @@
     public void addOutputAlias(String output, LogicalPlan cond) {
         mCondPlans.put(output, cond);
     }
-    
+*/
     public void addOutput(LogicalOperator lOp) {
         mOutputs.add(lOp);
     }

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
 (original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
 Tue May 20 14:41:07 2008
@@ -32,6 +32,7 @@
     private static final long serialVersionUID = 2L;
 
     protected int mIndex;
+    private LogicalPlan mCondPlan;
     
     /**
      * @param plan
@@ -43,9 +44,14 @@
      * @param conditions
      *            list of conditions for the split
      */
-    public LOSplitOutput(LogicalPlan plan, OperatorKey key, int index) {
+    public LOSplitOutput(LogicalPlan plan, OperatorKey key, int index, 
LogicalPlan condPlan) {
         super(plan, key);
         this.mIndex = index;
+        this.mCondPlan = condPlan;
+    }
+
+    public LogicalPlan getConditionPlan() {
+        return mCondPlan;
     }
     
     @Override

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java 
(original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java 
Tue May 20 14:41:07 2008
@@ -64,19 +64,16 @@
                 Iterator<LogicalOperator> iter = s.iterator();
                 LogicalOperator op = iter.next();
                 if (null == op) {
-                    log.debug("getSchema: Operator not in plan");
                     throw new FrontendException("Could not find operator in 
plan");
                 }
                 mSchema = op.getSchema();
-                log.debug("Printing aliases");
-                mSchema.printAliases();
                 while(iter.hasNext()) {
                     op = iter.next();
-                    if(null == mSchema) {
-                        log.debug("Schema is null, cannot perform schema 
merge");
-                        throw new FrontendException("Schema is null, cannot 
perform schema merge");
+                    if(null != mSchema) {
+                        mSchema = mSchema.merge(op.getSchema(), false);
+                    } else {
+                        mSchema = op.getSchema();
                     }
-                    mSchema = mSchema.merge(op.getSchema(), false);
                 }
                 mIsSchemaComputed = true;
             } catch (FrontendException fe) {

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
 (original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
 Tue May 20 14:41:07 2008
@@ -194,15 +194,8 @@
      */
     protected void visit(LOSplit split) throws VisitorException {
         // Visit each of split's conditions
-        for(LogicalPlan lp: split.getConditionPlans()) {
-            if (null != lp) {
-                PlanWalker w = new DependencyOrderWalker(lp);
-                pushWalker(w);
-                for(LogicalOperator logicalOp: lp.getRoots()) {
-                    logicalOp.visit(this);
-                }
-                popWalker();
-            }
+        for(LogicalOperator logicalOp: split.getOutputs()) {
+            logicalOp.visit(this);
         }
     }
 
@@ -295,7 +288,15 @@
     }
 
     protected void visit(LOSplitOutput sop) throws VisitorException {
-
+        LogicalPlan lp = sop.getConditionPlan();
+        if (null != lp) {
+            PlanWalker w = new DependencyOrderWalker(lp);
+            pushWalker(w);
+            for(LogicalOperator logicalOp: lp.getRoots()) {
+                logicalOp.visit(this);
+            }
+            popWalker();
+        }
     }
 
     protected void visit(LODistinct dt) throws VisitorException {

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
 (original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
 Tue May 20 14:41:07 2008
@@ -268,9 +268,9 @@
        }                                       
 
         void addSplitOutput(LogicalPlan lp, LOSplit splitOp, String alias, 
LogicalPlan condPlan, int index) throws PlanException{
-               LogicalOperator splitOutput = new LOSplitOutput(lp, new 
OperatorKey(scope, getNextId()), index);
+               LogicalOperator splitOutput = new LOSplitOutput(lp, new 
OperatorKey(scope, getNextId()), index, condPlan);
                splitOp.addOutput(splitOutput);
-               splitOp.addOutputAlias(alias, condPlan);
+               //splitOp.addOutputAlias(alias, condPlan);
                addAlias(alias, splitOutput);
                
                lp.add(splitOutput);
@@ -383,7 +383,7 @@
 TOKEN : { <EVAL : "eval"> }
 TOKEN : { <ASC : "asc"> }
 TOKEN : { <DESC : "desc"> }
-TOKEN : { <INT : "integer"> }
+TOKEN : { <INT : "int"> }
 TOKEN : { <LONG : "long"> }
 TOKEN : { <FLOAT : "float"> }
 TOKEN : { <DOUBLE : "double"> }
@@ -435,10 +435,7 @@
        )
        { 
                if(null != root) {
-                       if(null != root.getSchema()) {
-                               log.debug("Root: " + root.getClass().getName() 
+ " schema aliases");
-                               root.getSchema().printAliases();
-                       }
+                               log.debug("Root: " + root.getClass().getName() 
+ " schema: " + root.getSchema());
                }
                
                log.trace("Exiting Parse");
@@ -460,7 +457,7 @@
        (
        input = NestedExpr(lp) <INTO> 
        {
-               splitOp = new LOSplit(lp, input.getOperatorKey(), new 
ArrayList<LogicalOperator>(), new HashMap<String,LogicalPlan>());
+               splitOp = new LOSplit(lp, input.getOperatorKey(), new 
ArrayList<LogicalOperator>());
                lp.add(splitOp);
                log.debug("Adding operator " + splitOp.getClass().getName() + " 
to the logical plan");          
         lp.connect(input, splitOp);
@@ -1863,9 +1860,13 @@
 {
  (t = <INT>| t = <LONG>| t = <FLOAT>| t = <DOUBLE>| t = <CHARARRAY>| t = 
<BYTEARRAY>) 
  {
+    String typeName = t.image;
        log.debug(" nameToTypeMap: " + nameToTypeMap);
-       log.debug("t: " + t + " type: " + 
nameToTypeMap.get(t.image.toUpperCase()));
-       type = nameToTypeMap.get(t.image.toUpperCase());
+       log.debug("t: " + t + " type: " + 
nameToTypeMap.get(typeName.toUpperCase()));
+    if(typeName.equalsIgnoreCase("int")) {
+        typeName = "integer";
+    }
+       type = nameToTypeMap.get(typeName.toUpperCase());
        log.trace("Exiting BasicType");
        return type;
  }

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
 (original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
 Tue May 20 14:41:07 2008
@@ -856,8 +856,29 @@
             throw new AssertionError("LOSplitOutput can only have 1 input") ;
         }
 
+        LogicalOperator input = list.get(0);
+        LogicalPlan condPlan = op.getConditionPlan() ;
+
+        // Check that the inner plan has only 1 output port
+        if (!condPlan.isSingleLeafPlan()) {
+            String msg = "Split's cond plan can only have one output (leaf)" ;
+            msgCollector.collect(msg, MessageType.Error) ;
+            throw new VisitorException(msg) ;
+        }
+            
+        checkInnerPlan(condPlan, input) ;
+                 
+        byte innerCondType = condPlan.getLeaves().get(0).getType() ;
+        if (innerCondType != DataType.BOOLEAN) {
+            String msg = "Split's condition must evaluate to boolean" ;
+            msgCollector.collect(msg, MessageType.Error) ;
+            throw new VisitorException(msg) ;
+        }
+
+        op.setType(input.getType()) ; // This should be bag always
+
         try {
-            op.setSchema(list.get(0).getSchema()) ;
+            op.setSchema(input.getSchema()) ;
         } 
         catch (ParseException pe) {
             String msg = "Problem while reading schemas from"
@@ -1083,32 +1104,6 @@
         
         LogicalOperator input = inputList.get(0) ;
         
-        // Checking internal plans.
-        ArrayList<LogicalPlan> condPlans
-                        = new 
ArrayList<LogicalPlan>(split.getConditionPlans()) ;
-
-        for(int i=0;i < split.getConditionPlans().size(); i++) {
-            
-            LogicalPlan condPlan = condPlans.get(i) ;
-
-            // Check that the inner plan has only 1 output port
-            if (!condPlan.isSingleLeafPlan()) {
-                String msg = "Split's cond plan can only have one output 
(leaf)" ;
-                msgCollector.collect(msg, MessageType.Error) ;
-                throw new VisitorException(msg) ;
-            }
-            
-            checkInnerPlan(condPlan, input) ;
-                  
-            byte innerCondType = condPlan.getLeaves().get(0).getType() ;
-            if (innerCondType != DataType.BOOLEAN) {
-                String msg = "Split's condition must evaluate to boolean" ;
-                msgCollector.collect(msg, MessageType.Error) ;
-                throw new VisitorException(msg) ;
-            }
-                       
-        }         
-        
         split.setType(input.getType()) ; // This should be bag always
         
         try {

Modified: 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
 (original)
+++ 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
 Tue May 20 14:41:07 2008
@@ -742,17 +742,17 @@
     @Test
     public void testQuery64() {
         buildPlan("a = load 'a' as (name: chararray, details: tuple(age, gpa), 
mymap: map[]);");
-        buildPlan("c = load 'a' as (name, details: bag{mytuple: tuple(age: 
integer, gpa)});");
+        buildPlan("c = load 'a' as (name, details: bag{mytuple: tuple(age: 
int, gpa)});");
         buildPlan("b = group a by details;");
         String query = "d = foreach b generate group.age;";
         buildPlan(query);
                buildPlan("e = foreach a generate name, details;");
-               buildPlan("f = LOAD 'myfile' AS (garage: bag{tuple1: 
tuple(num_tools: integer)}, links: bag{tuple2: tuple(websites: chararray)}, 
page: bag{something_stupid: tuple(yeah_double: double)}, coordinates: 
bag{another_tuple: tuple(ok_float: float, bite_the_array: bytearray, 
bag_of_unknown: bag{})});");
+               buildPlan("f = LOAD 'myfile' AS (garage: bag{tuple1: 
tuple(num_tools: int)}, links: bag{tuple2: tuple(websites: chararray)}, page: 
bag{something_stupid: tuple(yeah_double: double)}, coordinates: 
bag{another_tuple: tuple(ok_float: float, bite_the_array: bytearray, 
bag_of_unknown: bag{})});");
     }
 
     @Test
     public void testQueryFail64() {
-        String query = "foreach (load 'myfile' as (col1, col2 : bag{age: 
integer})) generate col1 ;";
+        String query = "foreach (load 'myfile' as (col1, col2 : bag{age: 
int})) generate col1 ;";
         try {
                buildPlan(query);
         } catch (AssertionFailedError e) {
@@ -782,14 +782,14 @@
 
     @Test
     public void testQuery66() {
-        buildPlan("define myFunc = com.mycompany.myudfs.myfunc(integer i, 
chararray c);");
+        buildPlan("define myFunc = com.mycompany.myudfs.myfunc(int i, 
chararray c);");
         buildPlan("a = load 'input1' as (name, age, gpa);");
         buildPlan("b = foreach a generate myFunc($0, $1);");
        }
 
     @Test
     public void testQueryFail66() {
-        buildPlan("define myFunc = com.mycompany.myudfs.myfunc(integer i, 
chararray c);");
+        buildPlan("define myFunc = com.mycompany.myudfs.myfunc(int i, 
chararray c);");
         buildPlan("a = load 'input1' as (name, age, gpa);");
         try {
                buildPlan("b = foreach a generate myFunc($0, $1, $2);");
@@ -853,9 +853,9 @@
     @Test
     public void testQuery72() {
         buildPlan("split (load 'a') into x if $0 > '7', y if $0 < '7';");
-        buildPlan("b = foreach x generate (integer)$0;");
+        buildPlan("b = foreach x generate (int)$0;");
         buildPlan("c = foreach y generate (bag)$1;");
-        buildPlan("d = foreach y generate (integer)($1/2);");
+        buildPlan("d = foreach y generate (int)($1/2);");
     }
 
     @Test
@@ -865,6 +865,21 @@
         buildPlan("c = foreach y generate $0, ($0 matches 'yuri.*' ? $1 - 10 : 
$1);");
     }
 
+    @Test
+    public void testQuery74() {
+        buildPlan("a = load 'a' as (field1: int, field2: long);");
+        buildPlan("b = load 'a' as (field1: bytearray, field2: double);");
+        buildPlan("c = group a by field1, b by field1;");
+        buildPlan("d = cogroup a by ((field1+field2)*field1), b by field1;");
+    }
+
+    @Test
+    public void testQuery75() {
+        buildPlan("union (load 'a'), (load 'b'), (load 'c');");
+    }
+    
+    // Helper Functions
+    
     // Helper Functions
     // =================
     public LogicalPlan buildPlan(String query) {
@@ -914,6 +929,7 @@
                 }
             }
             */
+
             assertTrue(lp != null);
             return lp;
         } catch (IOException e) {

Modified: 
incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java?rev=658450&r1=658449&r2=658450&view=diff
==============================================================================
--- 
incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java
 (original)
+++ 
incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java
 Tue May 20 14:41:07 2008
@@ -1428,18 +1428,15 @@
         // split
         LOSplit split1 = new LOSplit(plan,
                                      genNewOperatorKey(),
-                                     new ArrayList<LogicalOperator>(),
-                                     new HashMap<String,LogicalPlan>());
+                                     new ArrayList<LogicalOperator>());
 
         // output1
-        LOSplitOutput splitOutput1 = new LOSplitOutput(plan, 
genNewOperatorKey(), 0) ;
+        LOSplitOutput splitOutput1 = new LOSplitOutput(plan, 
genNewOperatorKey(), 0, innerPlan1) ;
         split1.addOutput(splitOutput1);
-        split1.addOutputAlias("output1", innerPlan1);
 
         // output2
-        LOSplitOutput splitOutput2 = new LOSplitOutput(plan, 
genNewOperatorKey(), 1) ;
+        LOSplitOutput splitOutput2 = new LOSplitOutput(plan, 
genNewOperatorKey(), 1, innerPlan2) ;
         split1.addOutput(splitOutput2);
-        split1.addOutputAlias("output2", innerPlan2);
 
         plan.add(load1);
         plan.add(split1);
@@ -1557,18 +1554,15 @@
         // split
         LOSplit split1 = new LOSplit(plan,
                                      genNewOperatorKey(),
-                                     new ArrayList<LogicalOperator>(),
-                                     new HashMap<String,LogicalPlan>());
+                                     new ArrayList<LogicalOperator>());
 
         // output1
-        LOSplitOutput splitOutput1 = new LOSplitOutput(plan, 
genNewOperatorKey(), 0) ;
+        LOSplitOutput splitOutput1 = new LOSplitOutput(plan, 
genNewOperatorKey(), 0, innerPlan1) ;
         split1.addOutput(splitOutput1);
-        split1.addOutputAlias("output1", innerPlan1);
 
         // output2
-        LOSplitOutput splitOutput2 = new LOSplitOutput(plan, 
genNewOperatorKey(), 1) ;
+        LOSplitOutput splitOutput2 = new LOSplitOutput(plan, 
genNewOperatorKey(), 1, innerPlan2) ;
         split1.addOutput(splitOutput2);
-        split1.addOutputAlias("output2", innerPlan2);
 
         plan.add(load1);
         plan.add(split1);


Reply via email to