Added: 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestExperimentalRule.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestExperimentalRule.java?rev=903018&view=auto
==============================================================================
--- 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestExperimentalRule.java
 (added)
+++ 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestExperimentalRule.java
 Tue Jan 26 00:02:23 2010
@@ -0,0 +1,316 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test;
+
+import java.util.List;
+
+import org.apache.pig.experimental.plan.BaseOperatorPlan;
+import org.apache.pig.experimental.plan.Operator;
+import org.apache.pig.experimental.plan.OperatorPlan;
+import org.apache.pig.experimental.plan.PlanVisitor;
+import org.apache.pig.experimental.plan.optimizer.Rule;
+import org.apache.pig.experimental.plan.optimizer.Transformer;
+
+import junit.framework.TestCase;
+
+public class TestExperimentalRule extends TestCase {
+
+    private static class SillyRule extends Rule {
+    
+        public SillyRule(String n, OperatorPlan p) {
+            super(n, p);            
+        }
+        
+        @Override
+        public Transformer getNewTransformer() {                       
+            return null;
+        }
+
+        @Override
+        protected OperatorPlan buildPattern() {
+            // TODO Auto-generated method stub
+            return null;
+        }
+        
+    }
+    
+    private static class SillyPlan extends BaseOperatorPlan {
+            
+        SillyPlan() {
+            super();
+        }
+
+    }
+    
+    private static class OP extends Operator {
+        OP(String n, OperatorPlan p) {
+            super(n, p);           
+        }
+
+        public void accept(PlanVisitor v) {
+            
+        }
+    }
+    
+    private static class OP_Load extends OP {
+        OP_Load(String n, OperatorPlan p) {
+            super(n, p);            
+        }
+    }
+    
+    private static class OP_Filter extends OP {
+        OP_Filter(String n, OperatorPlan p) {
+            super(n, p);            
+        }
+    }
+    
+    private static class OP_Split extends OP {
+        OP_Split(String n, OperatorPlan p) {
+            super(n, p);            
+        }
+    }
+    
+    private static class OP_Store extends OP {
+        OP_Store(String n, OperatorPlan p) {
+            super(n, p);            
+        }
+    }
+    
+    private static class OP_Join extends OP {
+        OP_Join(String n, OperatorPlan p) {
+            super(n, p);            
+        }
+    }
+
+    
+    OperatorPlan plan = null;
+    
+    public void setUp() {
+        plan = new SillyPlan();
+        Operator l1 = new OP_Load("p1", plan);
+        plan.add(l1);
+        Operator l2 = new OP_Load("p2", plan);
+        plan.add(l2);
+        Operator j1 = new OP_Join("j1", plan);
+        plan.add(j1);
+        Operator f1 = new OP_Filter("f1", plan);
+        plan.add(f1);
+        Operator f2 = new OP_Filter("f2", plan);
+        plan.add(f2);
+        Operator t1 = new OP_Split("t1",plan);
+        plan.add(t1);
+        Operator f3 = new OP_Filter("f3", plan);
+        plan.add(f3);
+        Operator f4 = new OP_Filter("f4", plan);
+        plan.add(f4);
+        Operator s1 = new OP_Store("s1", plan);
+        plan.add(s1);
+        Operator s2 = new OP_Store("s2", plan);
+        plan.add(s2);
+        
+        // load --|-join - filter - filter - split |- filter - store
+        // load --|                                |- filter - store
+        plan.connect(l1, j1);
+        plan.connect(l2, j1);
+        plan.connect(j1, f1);
+        plan.connect(f1, f2);
+        plan.connect(f2, t1);
+        plan.connect(t1, f3);
+        plan.connect(t1, f4);
+        plan.connect(f3, s1);
+        plan.connect(f4, s2);
+    }
+    
+    public void testSingleNodeMatch() {
+        // search for Load 
+        OperatorPlan pattern = new SillyPlan();
+        pattern.add(new OP_Load("mmm", pattern));
+        
+        Rule r = new SillyRule("basic", pattern);
+        List<OperatorPlan> l = r.match(plan);
+        assertEquals(l.size(), 2);
+        
+        Operator m1 = l.get(0).getRoots().get(0);
+        assertTrue(m1.getName().equals("p1") || m1.getName().equals("p2"));
+        assertEquals(l.get(0).size(), 1);
+        
+        Operator m2 = l.get(1).getRoots().get(0);
+        assertTrue(m2.getName().equals("p1") || m2.getName().equals("p2"));
+        assertEquals(l.get(1).size(), 1);
+        assertNotSame(m1.getName(), m2.getName());
+       
+        // search for filter
+        pattern = new SillyPlan();
+        pattern.add(new OP_Filter("mmm",pattern));
+        r = new SillyRule("basic", pattern);
+        l = r.match(plan);
+        assertEquals(l.size(), 4);
+        
+        m1 = l.get(0).getRoots().get(0);
+        assertTrue(m1.getName().equals("f1") || m1.getName().equals("f2") 
+                || m1.getName().equals("f3") || m1.getName().equals("f4"));
+        assertEquals(l.get(0).size(), 1);
+        
+        m2 = l.get(1).getRoots().get(0);
+        assertTrue(m1.getName().equals("f1") || m1.getName().equals("f2") 
+                || m1.getName().equals("f3") || m1.getName().equals("f4"));
+        assertEquals(l.get(1).size(), 1);
+        assertNotSame(m1.getName(), m2.getName());
+        
+        // search for store
+        pattern = new SillyPlan();
+        pattern.add(new OP_Store("mmm",pattern));
+        r = new SillyRule("basic", pattern);
+        l = r.match(plan);
+        assertEquals(l.size(), 2);
+        
+        m1 = l.get(0).getRoots().get(0);
+        assertTrue(m1.getName().equals("s1") || m1.getName().equals("s2"));
+        assertEquals(l.get(0).size(), 1);
+        
+        m2 = l.get(1).getRoots().get(0);
+        assertTrue(m2.getName().equals("s1") || m2.getName().equals("s2"));
+        assertEquals(l.get(1).size(), 1);
+        assertNotSame(m1.getName(), m2.getName());
+        
+        // search for split
+        pattern = new SillyPlan();
+        pattern.add(new OP_Split("mmm",pattern));
+        r = new SillyRule("basic", pattern);
+        l = r.match(plan);
+        assertEquals(l.size(), 1);
+        
+        m1 = l.get(0).getRoots().get(0);
+        assertTrue(m1.getName().equals("t1"));
+        assertEquals(l.get(0).size(), 1);
+        
+        // search for join
+        pattern = new SillyPlan();
+        pattern.add(new OP_Join("mmm",pattern));
+        r = new SillyRule("basic", pattern);
+        l = r.match(plan);
+        assertEquals(l.size(), 1);
+        
+        m1 = l.get(0).getRoots().get(0);
+        assertTrue(m1.getName().equals("j1"));
+        assertEquals(l.get(0).size(), 1);
+      
+    }
+    
+    public void testTwoNodeMatch() {
+        // search for 2 Loads at the same time 
+        OperatorPlan pattern = new SillyPlan();
+        pattern.add(new OP_Load("mmm1", pattern));
+        pattern.add(new OP_Load("mmm2", pattern));
+        
+        Rule r = new SillyRule("basic", pattern);
+        List<OperatorPlan> l = r.match(plan);
+        assertEquals(l.size(), 1);
+        
+        assertEquals(l.get(0).getRoots().size(), 2);
+        assertEquals(l.get(0).getLeaves().size(), 2);
+        assertEquals(l.get(0).size(), 2);
+        
+        Operator m1 = l.get(0).getRoots().get(0);
+        assertTrue(m1.getName().equals("p1") || m1.getName().equals("p2"));
+        Operator m2 = l.get(0).getRoots().get(1);
+        assertTrue(m2.getName().equals("p1") || m2.getName().equals("p2"));    
   
+        assertNotSame(m1.getName(), m2.getName());
+       
+        
+        // search for join then filter
+        pattern = new SillyPlan();
+        Operator s1 = new OP_Join("mmm1", pattern);
+        Operator s2 = new OP_Filter("mmm2", pattern);
+        pattern.add(s1);
+        pattern.add(s2);        
+        pattern.connect(s1, s2);
+        
+        r = new SillyRule("basic", pattern);
+        l = r.match(plan);
+        assertEquals(l.size(), 1);
+        
+        assertEquals(l.get(0).getRoots().size(), 1);
+        assertEquals(l.get(0).getLeaves().size(), 1);
+        assertEquals(l.get(0).size(), 2);
+        
+        m1 = l.get(0).getRoots().get(0);
+        assertTrue(m1.getName().equals("j1"));
+        m2 = l.get(0).getLeaves().get(0);
+        assertTrue(m2.getName().equals("f1"));       
+       
+  
+        // search for filter, then store
+        pattern = new SillyPlan();
+        s1 = new OP_Filter("mmm1", pattern);
+        s2 = new OP_Store("mmm2", pattern);        
+        pattern.add(s1);
+        pattern.add(s2);           
+        pattern.connect(s1, s2);        
+        
+        r = new SillyRule("basic", pattern);
+        l = r.match(plan);
+        assertEquals(2, l.size());
+        
+        assertEquals(l.get(0).getRoots().size(), 1);
+        assertEquals(l.get(0).getLeaves().size(), 1);                     
+        
+        // search for 2 loads, then join
+        pattern = new SillyPlan();
+        s1 = new OP_Load("mmm1", pattern);
+        s2 = new OP_Load("mmm2", pattern);
+        Operator s3 = new OP_Join("jjj", pattern);
+        pattern.add(s1);
+        pattern.add(s2);
+        pattern.add(s3);
+        pattern.connect(s1, s3);
+        pattern.connect(s2, s3);
+        
+        r = new SillyRule("basic", pattern);
+        l = r.match(plan);
+        assertEquals(l.size(), 1);
+        
+        // search for split then 2 filters
+        pattern = new SillyPlan();
+        s1 = new OP_Split("mmm1", pattern);
+        s2 = new OP_Filter("mmm2", pattern);
+        s3 = new OP_Filter("mmm3", pattern);
+        pattern.add(s1);
+        pattern.add(s2);        
+        pattern.add(s3);
+        pattern.connect(s1, s2);
+        pattern.connect(s1, s3);
+        
+        r = new SillyRule("basic", pattern);
+        l = r.match(plan);
+        assertEquals(1, l.size());
+        
+        assertEquals(l.get(0).getRoots().size(), 1);
+        assertEquals(l.get(0).getLeaves().size(), 2);
+        assertEquals(l.get(0).size(), 3);
+        
+        m1 = l.get(0).getRoots().get(0);
+        assertTrue(m1.getName().equals("t1"));
+        m2 = l.get(0).getLeaves().get(0);
+        assertTrue(m2.getName().equals("f3") || m2.getName().equals("f4"));    
+        m2 = l.get(0).getLeaves().get(1);
+        assertTrue(m2.getName().equals("f3") || m2.getName().equals("f4"));    
+    }
+   
+}

Modified: 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSecondarySort.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSecondarySort.java?rev=903018&r1=903017&r2=903018&view=diff
==============================================================================
--- 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSecondarySort.java
 (original)
+++ 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSecondarySort.java
 Tue Jan 26 00:02:23 2010
@@ -366,6 +366,26 @@
         assertTrue(so.getDistinctChanged()==0);
     }
     
+    // See PIG-1193
+    public void testSortOptimization8() throws Exception{
+        // Sort desc, used in UDF twice
+        LogicalPlanTester planTester = new LogicalPlanTester() ;
+        planTester.buildPlan("A = LOAD 'input1' AS (a0);");
+        planTester.buildPlan("B = group A all;");
+        planTester.buildPlan("C = foreach B { D = order A by $0 desc; generate 
DIFF(D, D);};");
+        
+        LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
+        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
+        
+        SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
+        so.visit();
+        
+        assertTrue(so.getNumMRUseSecondaryKey()==1);
+        assertTrue(so.getNumSortRemoved()==2);
+        assertTrue(so.getDistinctChanged()==0);
+    }
+    
     public void testNestedDistinctEndToEnd1() throws Exception{
         File tmpFile1 = File.createTempFile("test", "txt");
         PrintStream ps1 = new PrintStream(new FileOutputStream(tmpFile1));

Modified: 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidator.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidator.java?rev=903018&r1=903017&r2=903018&view=diff
==============================================================================
--- 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidator.java
 (original)
+++ 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidator.java
 Tue Jan 26 00:02:23 2010
@@ -5485,6 +5485,79 @@
       LogicalPlan plan = planTester.buildPlan("c = foreach b generate 
(chararray)viewinfo#'pos' as position;") ;
 
         // validate
+        runTypeCheckingValidator(plan);
+        
+        checkLoaderInCasts(plan, "org.apache.pig.builtin.PigStorage");
+    }
+    
+    /**
+     * test various scenarios with two level map lookup
+     */
+    @Test
+    public void testTwolevelMapLookupLineage() throws Exception {
+        List<String[]> queries = new ArrayList<String[]>();
+        // CASE 1: LOAD -> FILTER -> FOREACH -> LIMIT -> STORE
+        queries.add(new String[] {"sds = LOAD '/my/data/location' " +
+                       "AS (simpleFields:map[], mapFields:map[], 
listMapFields:map[]);",
+                       "queries = FILTER sds BY 
mapFields#'page_params'#'query' " +
+                       "is NOT NULL;",
+                       "queries_rand = FOREACH queries GENERATE " +
+                       "(CHARARRAY) (mapFields#'page_params'#'query') AS 
query_string;",
+                       "queries_limit = LIMIT queries_rand 100;",
+                       "STORE queries_limit INTO 'out';"});     
+        // CASE 2: LOAD -> FOREACH -> FILTER -> LIMIT -> STORE
+        queries.add(new String[]{"sds = LOAD '/my/data/location'  " +
+                       "AS (simpleFields:map[], mapFields:map[], 
listMapFields:map[]);",
+                       "queries_rand = FOREACH sds GENERATE " +
+                       "(CHARARRAY) (mapFields#'page_params'#'query') AS 
query_string;",
+                       "queries = FILTER queries_rand BY query_string IS NOT 
null;",
+                       "queries_limit = LIMIT queries 100;",
+                       "STORE queries_limit INTO 'out';"});
+        // CASE 3: LOAD -> FOREACH -> FOREACH -> FILTER -> LIMIT -> STORE
+        queries.add(new String[]{"sds = LOAD '/my/data/location'  " +
+                       "AS (simpleFields:map[], mapFields:map[], 
listMapFields:map[]);",
+                       "params = FOREACH sds GENERATE " +
+                       "(map[]) (mapFields#'page_params') AS params;",
+                       "queries = FOREACH params " +
+                       "GENERATE (CHARARRAY) (params#'query') AS 
query_string;",
+                       "queries_filtered = FILTER queries BY query_string IS 
NOT null;",
+                       "queries_limit = LIMIT queries_filtered 100;",
+                       "STORE queries_limit INTO 'out';"});
+        // CASE 4: LOAD -> FOREACH -> FOREACH -> LIMIT -> STORE
+        queries.add(new String[]{"sds = LOAD '/my/data/location'  " +
+                       "AS (simpleFields:map[], mapFields:map[], 
listMapFields:map[]);",
+                       "params = FOREACH sds GENERATE" +
+                       " (map[]) (mapFields#'page_params') AS params;",
+                       "queries = FOREACH params GENERATE " +
+                       "(CHARARRAY) (params#'query') AS query_string;",
+                       "queries_limit = LIMIT queries 100;",
+                       "STORE queries_limit INTO 'out';"});
+        // CASE 5: LOAD -> FOREACH -> FOREACH -> FOREACH -> LIMIT -> STORE
+        queries.add(new String[]{"sds = LOAD '/my/data/location'  " +
+                "AS (simpleFields:map[], mapFields:map[], 
listMapFields:map[]);",
+                "params = FOREACH sds GENERATE " +
+                "(map[]) (mapFields#'page_params') AS params;",
+                "queries = FOREACH params GENERATE " +
+                "(CHARARRAY) (params#'query') AS query_string;",
+                "rand_queries = FOREACH queries GENERATE query_string as 
query;",
+                "queries_limit = LIMIT rand_queries 100;",
+                "STORE rand_queries INTO 'out';"});
+        
+        for (String[] query: queries) {
+            LogicalPlan lp = null;
+            for (String queryLine : query) {
+                lp = planTester.buildPlan(queryLine);    
+            }
+            
+            // validate
+            runTypeCheckingValidator(lp);
+            checkLoaderInCasts(lp, "org.apache.pig.builtin.PigStorage");
+            
+        }
+    }
+    
+    private void runTypeCheckingValidator(LogicalPlan plan) throws 
+    PlanValidationException {
         CompilationMessageCollector collector = new 
CompilationMessageCollector() ;
         TypeCheckingValidator typeValidator = new TypeCheckingValidator() ;
         typeValidator.validate(plan, collector) ;
@@ -5496,12 +5569,16 @@
         if (collector.hasError()) {
             throw new AssertionError("Expect no  error") ;
         }
-
+    }
+    
+    private void checkLoaderInCasts(LogicalPlan plan, String loaderClassName) 
+    throws VisitorException {
         CastFinder cf = new CastFinder(plan);
         cf.visit();
         List<LOCast> casts = cf.casts;
         for (LOCast cast : casts) {
-            
assertTrue(cast.getLoadFuncSpec().getClassName().startsWith("org.apache.pig.builtin.PigStorage"));
    
+            assertTrue(cast.getLoadFuncSpec().getClassName().startsWith(
+                    loaderClassName));    
         }
     }
     
@@ -5656,6 +5733,7 @@
      */
     public static class TestBinCondFieldSchema extends EvalFunc<DataBag> {
         //no-op exec method
+        @Override
         public DataBag exec(Tuple input) {
             return null;
         }


Reply via email to