Author: gates
Date: Mon Aug 11 10:34:02 2008
New Revision: 684831

URL: http://svn.apache.org/viewvc?rev=684831&view=rev
Log:
PIG-338. Daniel's patch to fix limit after distinct.


Modified:
    
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java
    
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java
    
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot
    
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot
    
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot

Modified: 
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=684831&r1=684830&r2=684831&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
 (original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
 Mon Aug 11 10:34:02 2008
@@ -679,7 +679,7 @@
                        log.warn("Something in the reduce plan while map plan 
is not done. Something wrong!");
                 }
             } else if (mro.isMapDone() && !mro.isReduceDone()) {
-               // limit should add into reduce reduce function
+               // limit should add into reduce plan
                 mro.reducePlan.addAsLeaf(op);
             } else {
                 log.warn("Both map and reduce phases have been done. This is 
unexpected while compiling!");

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java?rev=684831&r1=684830&r2=684831&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java
 (original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java
 Mon Aug 11 10:34:02 2008
@@ -97,7 +97,7 @@
         // Limit cannot be pushed up
         if (predecessor instanceof LOCogroup || predecessor instanceof 
LOFilter ||
                        predecessor instanceof LOLoad || predecessor instanceof 
LOSplit ||
-                       predecessor instanceof LOSplitOutput)
+                       predecessor instanceof LOSplitOutput || predecessor 
instanceof LODistinct)
         {
                return;
         }
@@ -129,19 +129,19 @@
                                {
                                        throw new OptimizerException("LOFilter 
should have one input");
                                }
+                   // we can move LOLimit even further, recursively optimize 
LOLimit
+                   processNode(limit);
                }
-               // we can move LOLimit even further, recursively optimize 
LOLimit
-               processNode(limit);
         }
         // Limit can be duplicated, and the new instance pushed in front of an 
operator for the following operators 
         // (that is, if you have X->limit, you can transform that to 
limit->X->limit):
-        else if (predecessor instanceof LOCross || predecessor instanceof 
LODistinct ||
+        else if (predecessor instanceof LOCross || 
                        predecessor instanceof LOForEach || predecessor 
instanceof LOUnion)
         {
                LOLimit newLimit = null;
                        // Process the predecessors with only one input. 
LOForEach should now have at least
                        // one flaten
-                       if (predecessor instanceof LODistinct || predecessor 
instanceof LOForEach)
+                       if (predecessor instanceof LOForEach)
                        {
                                LogicalOperator prepredecessor = 
mPlan.getPredecessors(predecessor).get(0);
                                try {

Modified: 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java?rev=684831&r1=684830&r2=684831&view=diff
==============================================================================
--- 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java 
(original)
+++ 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java 
Mon Aug 11 10:34:02 2008
@@ -69,7 +69,9 @@
         int len = fis.read(b);
         String goldenPlan = new String(b, 0, len);
         
-               String actualPlan = printLimitGraph(plan);              
+        String actualPlan = printLimitGraph(plan);
+        System.out.println("We get:");
+        System.out.println(actualPlan);
                assertEquals(goldenPlan, actualPlan);
     }
     
@@ -118,8 +120,9 @@
        // Duplicte limit with one input
        public void testOPLimit4Optimizer() throws Exception {
            planTester.buildPlan("A = load 'myfile1';");
-           planTester.buildPlan("B = distinct A;");
-           LogicalPlan plan = planTester.buildPlan("C = limit B 100;");
+           planTester.buildPlan("B = group A by $0;");
+           planTester.buildPlan("C = foreach B generate flatten(A);");
+           LogicalPlan plan = planTester.buildPlan("D = limit C 100;");
            optimizePlan(plan);
            compareWithGoldenFile(plan, FILE_BASE_LOCATION + 
"optlimitplan4.dot");
        }

Modified: 
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot?rev=684831&r1=684830&r2=684831&view=diff
==============================================================================
--- 
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot
 (original)
+++ 
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot
 Mon Aug 11 10:34:02 2008
@@ -1,7 +1,7 @@
 digraph graph1 {
-    LOLoad31 -> LOLimit34;
-    LOLimit34 [limit="100"];
-    LOLimit34 -> LODistinct32;
-    LODistinct32 -> LOLimit33;
-    LOLimit33 [limit="100"];
+    LOLoad31 -> LOCogroup33;
+    LOCogroup33 -> LOForEach36;
+    LOForEach36 [hasflat="true"];
+    LOForEach36 -> LOLimit37;
+    LOLimit37 [limit="100"];
 }
\ No newline at end of file

Modified: 
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot?rev=684831&r1=684830&r2=684831&view=diff
==============================================================================
--- 
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot
 (original)
+++ 
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot
 Mon Aug 11 10:34:02 2008
@@ -1,6 +1,6 @@
 digraph graph1 {
-    LOLoad35 -> LOLimit39;
-    LOLimit39 [limit="100"];
-    LOLimit39 -> LOForEach38;
-    LOForEach38 [hasflat="false"];
+    LOLoad38 -> LOLimit42;
+    LOLimit42 [limit="100"];
+    LOLimit42 -> LOForEach41;
+    LOForEach41 [hasflat="false"];
 }
\ No newline at end of file

Modified: 
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot?rev=684831&r1=684830&r2=684831&view=diff
==============================================================================
--- 
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot
 (original)
+++ 
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot
 Mon Aug 11 10:34:02 2008
@@ -1,4 +1,4 @@
 digraph graph1 {
-    LOLoad40 -> LOLimit41;
-    LOLimit41 [limit="20"];
+    LOLoad43 -> LOLimit44;
+    LOLimit44 [limit="20"];
 }
\ No newline at end of file


Reply via email to