Author: olga
Date: Mon Jul 28 12:46:00 2008
New Revision: 680460

URL: http://svn.apache.org/viewvc?rev=680460&view=rev
Log:
migration of PIG-154 to types brunch

Added:
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODefine.java
Modified:
    incubator/pig/branches/types/src/org/apache/pig/PigServer.java
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOStore.java
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
    incubator/pig/branches/types/src/org/apache/pig/tools/grunt/GruntParser.java
    
incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
    incubator/pig/branches/types/test/org/apache/pig/test/TestGrunt.java
    
incubator/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java
    
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java

Modified: incubator/pig/branches/types/src/org/apache/pig/PigServer.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/PigServer.java?rev=680460&r1=680459&r2=680460&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/PigServer.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/PigServer.java Mon Jul 28 
12:46:00 2008
@@ -63,7 +63,8 @@
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.util.WrappedIOException;
 import org.apache.pig.impl.util.PropertiesUtil;
-
+import org.apache.pig.impl.logicalLayer.LODefine;
+import org.apache.pig.impl.logicalLayer.LOStore;
 
 /**
  * 
@@ -242,12 +243,31 @@
         }
             
         LogicalPlan lp = null;
+        LogicalOperator op = null;
         try {
             lp = (new LogicalPlanBuilder(pigContext).parse(scope, query,
                     aliases, opTable, aliasOp, startLine));
         } catch (ParseException e) {
             throw (IOException) new IOException(e.getMessage()).initCause(e);
         }
+        
+        if (lp.getLeaves().size() == 1)
+        {
+            op = lp.getSingleLeafPlanOutputOp();
+            // No need to do anything about DEFINE 
+            if (op instanceof LODefine) {
+                return;
+            }
+        
+            // Check if we just processed a LOStore i.e. STORE
+            if (op instanceof LOStore) {
+                try{
+                    execute(lp);
+                } catch (Exception e) {
+                    throw WrappedIOException.wrap("Unable to store for alias: 
" + op.getOperatorKey().getId(), e);
+                }
+            }
+        }
     }
 
     public void registerQuery(String query) throws IOException {

Added: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODefine.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODefine.java?rev=680460&view=auto
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODefine.java 
(added)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODefine.java 
Mon Jul 28 12:46:00 2008
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.impl.logicalLayer;
+import org.apache.pig.impl.plan.OperatorKey;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+public class LODefine extends LogicalOperator {
+    private static final long serialVersionUID = 1L;
+
+    public LODefine(LogicalPlan plan, OperatorKey key){
+        super(plan, key);
+    }
+
+    public Schema getSchema() throws FrontendException {
+        return null;
+    }
+    
+    public void visit(LOVisitor v) {}
+
+    public String name() {
+        return "Define " + mKey.scope + "-" + mKey.id;
+    }
+
+    public boolean supportsMultipleInputs(){
+        return false;
+    }
+
+}

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOStore.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOStore.java?rev=680460&r1=680459&r2=680460&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOStore.java 
(original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOStore.java 
Mon Jul 28 12:46:00 2008
@@ -82,9 +82,10 @@
     }
 
     @Override
-    public Schema getSchema() throws RuntimeException {
-        throw new RuntimeException("Internal error: Requested schema of a "
-                + "store operator.");
+    public Schema getSchema() throws FrontendException {
+        //throw new RuntimeException("Internal error: Requested schema of a "
+         //       + "store operator.");
+        return mPlan.getPredecessors(this).get(0).getSchema();
     }
 
     @Override

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=680460&r1=680459&r2=680460&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
 (original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
 Mon Jul 28 12:46:00 2008
@@ -145,6 +145,8 @@
         }
            
         aliases.put(store, storePlan);
+
+        if (storePlan.getRoots().size() == 0) throw new 
RuntimeException("Store plan has no roots!");
         return storePlan;
     }
 
@@ -493,6 +495,7 @@
 TOKEN : { <FILTEROP : <STRFILTEROP> | <NUMFILTEROP>  > }
 
 // List all the keywords in the language
+TOKEN : { <DEFINE : "define"> }
 TOKEN : { <LOAD : "load"> }
 TOKEN : { <FILTER : "filter"> }
 TOKEN : { <FOREACH : "foreach"> }
@@ -536,6 +539,7 @@
 TOKEN : { <MAP : "map"> }
 TOKEN : { <IS : "is"> }
 TOKEN : { <NULL : "null"> }
+TOKEN : { <STORE : "store"> }
 TOKEN : { <LIMIT : "limit"> }
 
 TOKEN:
@@ -727,7 +731,8 @@
 {
        (
        (
-       (<LOAD> op = LoadClause(lp) [<AS> (LOOKAHEAD(2) "(" schema = 
TupleSchema() ")" {op.setSchema(schema); op.setCanonicalNames(); 
log.debug("Load as schema()");schema.printAliases();} | fs = AtomSchema() 
{schema = new Schema(fs); op.setSchema(schema); log.debug("Load as 
atomschema()");schema.printAliases();}) ])
+    (<DEFINE> op = DefineClause(lp))
+|      (<LOAD> op = LoadClause(lp) [<AS> (LOOKAHEAD(2) "(" schema = 
TupleSchema() ")" {op.setSchema(schema); op.setCanonicalNames(); 
log.debug("Load as schema()");schema.printAliases();} | fs = AtomSchema() 
{schema = new Schema(fs); op.setSchema(schema); log.debug("Load as 
atomschema()");schema.printAliases();}) ])
 |      ((<GROUP> | <COGROUP>) op = CogroupClause(lp))
 |      (<FILTER> op = FilterClause(lp))
 |   (<LIMIT> op = LimitClause(lp))
@@ -745,6 +750,7 @@
 |   (<JOIN> op = JoinClause(lp))
 |      (<UNION> op = UnionClause(lp))
 |      (<FOREACH> op = ForEachClause(lp))
+|   (<STORE> op = StoreClause(lp))
        )
     [<PARALLEL> t2=<INTEGER> { 
op.setRequestedParallelism(Integer.parseInt(t2.image));} ]
        )       
@@ -1469,6 +1475,52 @@
        }
 }
 
+LogicalOperator DefineClause(LogicalPlan lp) : {Token t; Token t1; String 
functionName, functionArgs;}
+{
+    t = <IDENTIFIER>
+    (
+        functionName = QualifiedFunction() "(" functionArgs = StringList() ")"
+        {
+            pigContext.registerFunction(t.image, new FuncSpec(functionName + 
"(" + functionArgs + ")"));
+        }
+    )
+    {
+        // Return the dummy LODefine
+        LogicalOperator lo = new LODefine(lp, new OperatorKey(scope, 
getNextId()));
+        lp.add(lo);
+        return lo;
+    }
+}
+
+LogicalOperator StoreClause(LogicalPlan lp) : {LogicalOperator lo; Token t; 
String fileName; String functionSpec = null; 
+                                                String functionName, 
functionArgs;}
+{
+    t = <IDENTIFIER> <INTO> fileName = FileName()
+    (
+        <USING> functionName = QualifiedFunction() {functionSpec = 
functionName;}
+        (
+            "(" functionArgs = StringList() ")" {functionSpec = functionSpec + 
"(" + functionArgs + ")";}
+        )?
+    )?
+    {
+
+        if (functionSpec == null)
+            functionSpec = PigStorage.class.getName() + "()";
+
+        LogicalOperator store = new LOStore(lp, new OperatorKey(scope, 
getNextId()),
+                                            new FileSpec(fileName, new 
FuncSpec(functionSpec)));
+
+        LogicalOperator input = mapAliasOp.get(t.image);
+        if (input == null)
+            throw new ParseException("Unable to find alias " + t.image);
+
+        lp.add(store);
+        lp.add(input);
+        lp.connect(input, store);
+        return store;
+    }
+}
+
 ArrayList<LogicalOperator> NestedBlock(Schema over, ArrayList<LogicalOperator> 
specList, LogicalPlan lp, LogicalOperator input):
 {
        LogicalOperator spec; 

Modified: 
incubator/pig/branches/types/src/org/apache/pig/tools/grunt/GruntParser.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/grunt/GruntParser.java?rev=680460&r1=680459&r2=680460&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/tools/grunt/GruntParser.java 
(original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/tools/grunt/GruntParser.java 
Mon Jul 28 12:46:00 2008
@@ -145,10 +145,6 @@
         mDone = true;
     }
     
-    protected void processRegisterFunc(String name, String expr) {
-        mPigServer.registerFunction(name, new FuncSpec(expr));
-    }
-    
     protected void processDescribe(String alias) throws IOException {
         mPigServer.dumpSchema(alias);
     }
@@ -185,10 +181,6 @@
         }
     }
     
-    protected void processStore(String alias, String file, String func) throws 
IOException {
-        mPigServer.store(alias, file, func);
-    }
-        
     protected void processCat(String path) throws IOException
     {
         try {

Modified: 
incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=680460&r1=680459&r2=680460&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
 (original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
 Mon Jul 28 12:46:00 2008
@@ -59,8 +59,6 @@
        
        abstract protected void quit();
        
-       abstract protected void processRegisterFunc(String name, String expr);
-       
        abstract protected void processDescribe(String alias) throws 
IOException;
 
     abstract protected void processExplain(String alias) throws IOException;
@@ -68,8 +66,6 @@
        abstract protected void processRegister(String jar) throws IOException;
 
        abstract protected void processSet(String key, String value) throws 
IOException, ParseException;
-       
-       abstract protected void processStore(String alias, String file, String 
func) throws IOException;
                
        abstract protected void processCat(String path) throws IOException;
 
@@ -127,7 +123,6 @@
 TOKEN: {<COPY: "cp">}
 TOKEN: {<COPYFROMLOCAL: "copyFromLocal">}
 TOKEN: {<COPYTOLOCAL: "copyToLocal">}
-TOKEN: {<DEFINE: "define">}
 TOKEN: {<DUMP: "dump">}
 TOKEN: {<DESCRIBE: "describe">}
 TOKEN: {<EXPLAIN: "explain">}
@@ -141,15 +136,12 @@
 TOKEN: {<REGISTER: "register">}
 TOKEN: {<REMOVE: "rm">}
 TOKEN: {<SET: "set">}
-TOKEN: {<STORE: "store">}
-TOKEN: {<INTO: "into">}
-TOKEN: {<USING: "using">}
 
 // internal use commands
 TOKEN: {<SCRIPT_DONE: "scriptDone">}
 
 // Define pig command as 
-// (1) Starting with "split" or assignment (A=) followed by
+// (1) Starting with "split"/"define"/"store" or assignment (A=) followed by
 // (2) Single statement followed by ; and newline or
 // (3) Block of statements enclosed in
 
@@ -173,8 +165,9 @@
 
 <DEFAULT> MORE :
 {
-       <"split"> : PIG_START
-|      <"("> : FUNC_ARGS_START
+    <"split"> : PIG_START
+|   <"define"> : PIG_START
+|      <"store"> : PIG_START
 |      <(["a"-"z", "A"-"Z"])+(["a"-"z", "A"-"Z"] | ["0"-"9"] | "_")*(" " | 
"\t")*"="> : PIG_START
 }
 
@@ -230,25 +223,6 @@
                }: DEFAULT
 }
 
-<FUNC_ARGS_START> MORE :
-{
-       <"("> {funcBlockLevel = 1;} : IN_ARG_BLOCK
-|      <")"> : FUNC_ARGS_END
-|      <(~[])>
-}
-
-<IN_ARG_BLOCK> MORE:
-{
-        <"("> {funcBlockLevel++;}
-|       <")"> {funcBlockLevel--; if (funcBlockLevel == 0) 
SwitchTo(FUNC_ARGS_END);}
-|       <(~[])>
-}
-
-<FUNC_ARGS_END> TOKEN :
-{
-        <FUNC_ARGS: ""> {matchedToken.image = image.toString();}: DEFAULT
-}
-
 // other
 TOKEN: {<EOL:  "\r" | "\n" | "\r\n">}
 TOKEN: {<QUOTE: "'">}
@@ -319,13 +293,6 @@
        t2 = GetPath()
        {processCopyToLocal(t1.image, t2.image);}       
        |
-       <DEFINE>
-       t1 = <IDENTIFIER>
-       (
-       val = QualifiedFunction()
-       )
-       {processRegisterFunc(t1.image, val);}
-       |
        <DUMP>
        t1 = <IDENTIFIER>
        {processDump(t1.image);}
@@ -391,16 +358,6 @@
                {processSet(t1.image, t2.image);}
        )
        |
-       <STORE>
-       t1 = <IDENTIFIER>
-       <INTO>
-       t2 = GetPath()
-       (
-               <USING>
-               val = QualifiedFunction()
-       )?
-       {processStore(t1.image, unquote(t2.image), val);}
-       |
        <EOF>
        {quit();}
        |
@@ -413,17 +370,6 @@
        )
 }
 
-String QualifiedFunction()       : {Token t1;StringBuffer s=new 
StringBuffer();}
-{
-       t1 = GetPath()
-       {s.append(t1.image);}
-       (
-               t1 = <FUNC_ARGS>
-               {s.append(t1.image);}
-       )*
-        {return s.toString();}
-}
-
 Token GetPath() :
 {
        Token t;
@@ -478,8 +424,6 @@
        |
        t = <COPYTOLOCAL>
        |
-       t = <DEFINE>
-       |
        t = <DUMP>
        |
        t = <DESCRIBE>
@@ -506,12 +450,6 @@
        |
        t = <SET>
        |
-       t = <STORE>
-       |
-       t = <INTO>
-       |
-       t = <USING>
-       |
        t = <SCRIPT_DONE>
        )
 

Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestGrunt.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestGrunt.java?rev=680460&r1=680459&r2=680460&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestGrunt.java 
(original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestGrunt.java Mon 
Jul 28 12:46:00 2008
@@ -37,7 +37,7 @@
 
     private final Log log = LogFactory.getLog(getClass());
     
-    @Test 
+/*    @Test 
     public void testCopyFromLocal() throws Throwable {
         PigServer server = new PigServer(ExecType.MAPREDUCE, 
cluster.getProperties());
         PigContext context = server.getPigContext();
@@ -50,14 +50,14 @@
         Grunt grunt = new Grunt(new BufferedReader(reader), context);
     
         grunt.exec();
-    }
+    }*/
 
     @Test 
     public void testDefine() throws Throwable {
         PigServer server = new PigServer("MAPREDUCE");
         PigContext context = server.getPigContext();
         
-        String strCmd = "define myudf org.apache.pig.builtin.AVG();";
+        String strCmd = "define myudf org.apache.pig.builtin.AVG();\n";
         
         ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
         InputStreamReader reader = new InputStreamReader(cmd);

Modified: 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java?rev=680460&r1=680459&r2=680460&view=diff
==============================================================================
--- 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java 
(original)
+++ 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java 
Mon Jul 28 12:46:00 2008
@@ -40,6 +40,7 @@
 import 
org.apache.pig.backend.hadoop.executionengine.physicalLayer.LogToPhyTranslationVisitor;
 import org.apache.pig.impl.logicalLayer.LogicalOperator;
 import org.apache.pig.impl.logicalLayer.LOLoad;
+import org.apache.pig.impl.logicalLayer.LODefine;
 import org.apache.pig.impl.logicalLayer.LogicalPlan;
 import org.apache.pig.impl.logicalLayer.LogicalPlanBuilder;
 import org.apache.pig.impl.plan.OperatorKey;
@@ -580,8 +581,8 @@
             
             if(roots.size() > 0) {
                 for(LogicalOperator op: roots) {
-                    if (!(op instanceof LOLoad)){
-                        throw new Exception("Cannot have a root that is not 
the load operator LOLoad. Found " + op.getClass().getName());
+                    if (!(op instanceof LOLoad) && !(op instanceof LODefine)){
+                        throw new Exception("Cannot have a root that is not 
the load or define operator. Found " + op.getClass().getName());
                     }
                 }
             }

Modified: 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=680460&r1=680459&r2=680460&view=diff
==============================================================================
--- 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
 (original)
+++ 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
 Mon Jul 28 12:46:00 2008
@@ -48,7 +48,7 @@
 import org.apache.pig.impl.plan.OperatorKey;
 import org.apache.pig.impl.logicalLayer.LOCogroup;
 import org.apache.pig.impl.logicalLayer.LOLoad;
-//import org.apache.pig.impl.logicalLayer.LOEval;
+import org.apache.pig.impl.logicalLayer.LODefine;
 import org.apache.pig.impl.logicalLayer.*;
 import org.apache.pig.impl.logicalLayer.ExpressionOperator;
 import org.apache.pig.impl.logicalLayer.LogicalPlan;
@@ -145,6 +145,37 @@
         buildPlan(query);
     }
     
+    @Test
+    public void testQuery100() {
+        // test define syntax
+        String query = "define FUNC ARITY();";
+        LogicalOperator lo = buildPlan(query).getRoots().get(0);
+        assertTrue(lo instanceof LODefine);
+    }
+
+    @Test
+    public void testQuery101() {
+        // test usage of an alias from define
+        String query = "define FUNC ARITY();";
+        buildPlan(query);
+
+        query = "foreach (load 'data') generate FUNC($0);";
+        buildPlan(query);
+    }
+
+    @Test
+    public void testQuery102() {
+        // test basic store
+        buildPlan("a = load 'a';");
+        printPlan(buildPlan("store a into 'out';"));
+    }
+
+    @Test
+    public void testQuery103() {
+        // test store with store function
+        buildPlan("a = load 'a';");
+        buildPlan("store a into 'out' using PigStorage();");
+    }
 
     @Test
     public void testQueryFail1() {
@@ -1104,7 +1135,6 @@
 
     public LogicalPlan buildPlan(String query, ClassLoader cldr) {
         LogicalPlanBuilder.classloader = cldr;
-        PigContext pigContext = new PigContext(ExecType.LOCAL, new 
Properties());
         LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext); //
 
         try {
@@ -1117,8 +1147,8 @@
             
             if(roots.size() > 0) {
                 for(LogicalOperator op: roots) {
-                    if (!(op instanceof LOLoad)){
-                        throw new Exception("Cannot have a root that is not 
the load operator LOLoad. Found " + op.getClass().getName());
+                    if (!(op instanceof LOLoad) && !(op instanceof LODefine)){
+                        throw new Exception("Cannot have a root that is not 
the load or define operator. Found " + op.getClass().getName());
                     }
                 }
             }
@@ -1162,4 +1192,5 @@
     Map<LogicalOperator, LogicalPlan> aliases = new HashMap<LogicalOperator, 
LogicalPlan>();
     Map<OperatorKey, LogicalOperator> logicalOpTable = new 
HashMap<OperatorKey, LogicalOperator>();
     Map<String, LogicalOperator> aliasOp = new HashMap<String, 
LogicalOperator>();
+    PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties());
 }


Reply via email to