Author: olga
Date: Mon Aug 18 17:09:16 2008
New Revision: 686902

URL: http://svn.apache.org/viewvc?rev=686902&view=rev
Log:
PIG-362: don't push limit above generate with flatten

Added:
    
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan7.dot
Modified:
    incubator/pig/branches/types/CHANGES.txt
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java
    
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java

Modified: incubator/pig/branches/types/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/CHANGES.txt?rev=686902&r1=686901&r2=686902&view=diff
==============================================================================
--- incubator/pig/branches/types/CHANGES.txt (original)
+++ incubator/pig/branches/types/CHANGES.txt Mon Aug 18 17:09:16 2008
@@ -159,3 +159,5 @@
     PIG-301: fix for order by descending
 
     PIG-378: fix for GENERATE + LIMIT
+
+    PIG-362: don't push limit above generate with flatten

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java?rev=686902&r1=686901&r2=686902&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java
 (original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java
 Mon Aug 18 17:09:16 2008
@@ -135,17 +135,16 @@
         }
         // Limit can be duplicated, and the new instance pushed in front of an 
operator for the following operators 
         // (that is, if you have X->limit, you can transform that to 
limit->X->limit):
-        else if (predecessor instanceof LOCross || 
-                       predecessor instanceof LOForEach || predecessor 
instanceof LOUnion)
+        else if (predecessor instanceof LOCross || predecessor instanceof 
LOUnion)
         {
                LOLimit newLimit = null;
-                       // Process the predecessors with only one input. 
LOForEach should now have at least
-                       // one flaten
-                       if (predecessor instanceof LOForEach)
+                       List<LogicalOperator> nodesToProcess = new 
ArrayList<LogicalOperator>();
+                       for (LogicalOperator 
prepredecessor:mPlan.getPredecessors(predecessor))
+                               nodesToProcess.add(prepredecessor);
+                       for (LogicalOperator prepredecessor:nodesToProcess)
                        {
-                               LogicalOperator prepredecessor = 
mPlan.getPredecessors(predecessor).get(0);
                                try {
-                                       newLimit = limit.duplicate();
+                                       newLimit = (LOLimit)limit.duplicate();
                                        insertBetween(prepredecessor, newLimit, 
predecessor, null);
                                } catch (Exception e) {
                                        throw new OptimizerException("Can not 
insert LOLimit clone", e);
@@ -153,24 +152,6 @@
                                // we can move the new LOLimit even further, 
recursively optimize LOLimit
                                processNode(newLimit);
                        }
-                       // Now comes predecessors with multiple inputs
-                       else if (predecessor instanceof LOCross || predecessor 
instanceof LOUnion)
-                       {
-                               List<LogicalOperator> nodesToProcess = new 
ArrayList<LogicalOperator>();
-                               for (LogicalOperator 
prepredecessor:mPlan.getPredecessors(predecessor))
-                                       nodesToProcess.add(prepredecessor);
-                               for (LogicalOperator 
prepredecessor:nodesToProcess)
-                               {
-                                       try {
-                                               newLimit = 
(LOLimit)limit.duplicate();
-                                               insertBetween(prepredecessor, 
newLimit, predecessor, null);
-                                       } catch (Exception e) {
-                                               throw new 
OptimizerException("Can not insert LOLimit clone", e);
-                                       }
-                                       // we can move the new LOLimit even 
further, recursively optimize LOLimit
-                                       processNode(newLimit);
-                               }
-                       }
         }
         // Limit can be merged into LOSort, result a "limited sort"
         else if (predecessor instanceof LOSort)

Modified: 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java?rev=686902&r1=686901&r2=686902&view=diff
==============================================================================
--- 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java 
(original)
+++ 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java 
Mon Aug 18 17:09:16 2008
@@ -147,5 +147,15 @@
            optimizePlan(plan);
            compareWithGoldenFile(plan, FILE_BASE_LOCATION + 
"optlimitplan6.dot");
        }
+    
+    @Test
+    // Limit stay the same for ForEach with a flatten
+    public void testOPLimit7Optimizer() throws Exception {
+        planTester.buildPlan("A = load 'myfile1';");
+        planTester.buildPlan("B = foreach A generate flatten($0);");
+        LogicalPlan plan = planTester.buildPlan("C = limit B 100;");
+        optimizePlan(plan);
+        compareWithGoldenFile(plan, FILE_BASE_LOCATION + "optlimitplan7.dot");
+    }
 }
 

Added: 
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan7.dot
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan7.dot?rev=686902&view=auto
==============================================================================
--- 
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan7.dot
 (added)
+++ 
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan7.dot
 Mon Aug 18 17:09:16 2008
@@ -0,0 +1,6 @@
+digraph graph1 {
+    LOLoad47 -> LOForEach50;
+    LOForEach50 [hasflat="true"];
+    LOForEach50 -> LOLimit51;
+    LOLimit51 [limit="100"];
+}
\ No newline at end of file


Reply via email to