Author: gates
Date: Wed May  7 10:12:11 2008
New Revision: 654172

URL: http://svn.apache.org/viewvc?rev=654172&view=rev
Log:
PIG-202: Fix Order by so that user provided comparator func is used for 
quantile determination.


Added:
    
incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/InstantiateFuncCallerPOVisitor.java
Modified:
    incubator/pig/trunk/CHANGES.txt
    
incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapreducePlanCompiler.java
    
incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java
    incubator/pig/trunk/src/org/apache/pig/impl/builtin/FindQuantiles.java
    incubator/pig/trunk/src/org/apache/pig/impl/eval/BinCondSpec.java
    incubator/pig/trunk/src/org/apache/pig/impl/eval/CompositeEvalSpec.java
    incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpec.java
    incubator/pig/trunk/src/org/apache/pig/impl/eval/FilterSpec.java
    incubator/pig/trunk/src/org/apache/pig/impl/eval/FuncEvalSpec.java
    incubator/pig/trunk/src/org/apache/pig/impl/eval/GenerateSpec.java
    incubator/pig/trunk/src/org/apache/pig/impl/eval/SortDistinctSpec.java

Modified: incubator/pig/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=654172&r1=654171&r2=654172&view=diff
==============================================================================
--- incubator/pig/trunk/CHANGES.txt (original)
+++ incubator/pig/trunk/CHANGES.txt Wed May  7 10:12:11 2008
@@ -276,3 +276,5 @@
     PIG-229: Proper error handling in case of deserializer failure
 
        PIG-219: Change unit tests to run both local and map reduce modes (kali 
via gates).
+
+       PIG-202: Fix Order by so that user provided comparator func is used for 
quantile determination (kali via gates).

Modified: 
incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapreducePlanCompiler.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapreducePlanCompiler.java?rev=654172&r1=654171&r2=654172&view=diff
==============================================================================
--- 
incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapreducePlanCompiler.java
 (original)
+++ 
incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapreducePlanCompiler.java
 Wed May  7 10:12:11 2008
@@ -353,7 +353,9 @@
         
         //sort the first column of the cogroup output and feed it to the 
quantiles function
         EvalSpec sortedSampleSpec = new ProjectSpec(1);
-        sortedSampleSpec = sortedSampleSpec.addSpec(new 
SortDistinctSpec(false, new StarSpec()));
+        EvalSpec starSpec = new StarSpec();
+        starSpec.setComparatorName(loSort.getSortSpec().getComparatorName());
+        sortedSampleSpec = sortedSampleSpec.addSpec(new 
SortDistinctSpec(false, starSpec));
         argsList.add(sortedSampleSpec);
         
         EvalSpec args = new GenerateSpec(argsList);

Added: 
incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/InstantiateFuncCallerPOVisitor.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/InstantiateFuncCallerPOVisitor.java?rev=654172&view=auto
==============================================================================
--- 
incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/InstantiateFuncCallerPOVisitor.java
 (added)
+++ 
incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/InstantiateFuncCallerPOVisitor.java
 Wed May  7 10:12:11 2008
@@ -0,0 +1,62 @@
+/**
+ * 
+ */
+package org.apache.pig.backend.local.executionengine;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.pig.backend.executionengine.ExecPhysicalOperator;
+import org.apache.pig.backend.hadoop.executionengine.POMapreduce;
+import org.apache.pig.impl.FunctionInstantiator;
+import org.apache.pig.impl.eval.EvalSpec;
+import org.apache.pig.impl.logicalLayer.OperatorKey;
+import org.apache.pig.impl.physicalLayer.POVisitor;
+
+/**
+ * POVisitor calling instantiateFunc on all EvalSpec members of visited nodes.
+ */
+public class InstantiateFuncCallerPOVisitor extends POVisitor {
+
+    private FunctionInstantiator instantiator;
+
+    protected InstantiateFuncCallerPOVisitor(FunctionInstantiator instantiator,
+            Map<OperatorKey, ExecPhysicalOperator> opTable) {
+        super(opTable);
+        this.instantiator = instantiator;
+    }
+
+    private void callInstantiateFunc(EvalSpec spec) {
+        try {
+            spec.instantiateFunc(instantiator);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void visitCogroup(POCogroup g) {
+        super.visitCogroup(g);
+        for (EvalSpec es : g.specs)
+            callInstantiateFunc(es);
+    }
+
+    @Override
+    public void visitMapreduce(POMapreduce mr) {
+        super.visitMapreduce(mr);
+        for (EvalSpec es : mr.groupFuncs)
+            callInstantiateFunc(es);
+    }
+
+    @Override
+    public void visitSort(POSort s) {
+        super.visitSort(s);
+        callInstantiateFunc(s.sortSpec);
+    }
+
+    @Override
+    public void visitEval(POEval e) {
+        super.visitEval(e);
+        callInstantiateFunc(e.spec);
+    }
+}

Modified: 
incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java?rev=654172&r1=654171&r2=654172&view=diff
==============================================================================
--- 
incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java
 (original)
+++ 
incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java
 Wed May  7 10:12:11 2008
@@ -122,6 +122,7 @@
         try {
             PhysicalOperator pp = 
(PhysicalOperator)physicalOpTable.get(plan.getRoot());
 
+            pp.visit(new InstantiateFuncCallerPOVisitor(pigContext, 
physicalOpTable));
             pp.open();
             
             Tuple t;

Modified: incubator/pig/trunk/src/org/apache/pig/impl/builtin/FindQuantiles.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/builtin/FindQuantiles.java?rev=654172&r1=654171&r2=654172&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/builtin/FindQuantiles.java 
(original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/builtin/FindQuantiles.java Wed 
May  7 10:12:11 2008
@@ -39,15 +39,13 @@
         
         long numSamples = samples.size();
         
-        long toSkip = numSamples / numQuantiles;
-        
-        long i=0, nextQuantile = 0;
+        long i=0, written = 0;
         Iterator<Tuple> iter = samples.iterator();
         while (iter.hasNext()){
             Tuple t = iter.next();
-            if (i==nextQuantile){
+            if ((written + 1) * numSamples < i * (numQuantiles + 1)) {
                 output.add(t);
-                nextQuantile+=toSkip+1;
+                written++;
             }
             i++;
         }

Modified: incubator/pig/trunk/src/org/apache/pig/impl/eval/BinCondSpec.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/eval/BinCondSpec.java?rev=654172&r1=654171&r2=654172&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/eval/BinCondSpec.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/eval/BinCondSpec.java Wed May  
7 10:12:11 2008
@@ -58,6 +58,7 @@
     
     @Override
     public void instantiateFunc(FunctionInstantiator fInstantiaor) throws 
IOException{
+        super.instantiateFunc(fInstantiaor);
         cond.instantiateFunc(fInstantiaor);
         ifTrue.instantiateFunc(fInstantiaor);
         ifFalse.instantiateFunc(fInstantiaor);

Modified: 
incubator/pig/trunk/src/org/apache/pig/impl/eval/CompositeEvalSpec.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/eval/CompositeEvalSpec.java?rev=654172&r1=654171&r2=654172&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/eval/CompositeEvalSpec.java 
(original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/eval/CompositeEvalSpec.java Wed 
May  7 10:12:11 2008
@@ -102,9 +102,9 @@
     @Override
     public void instantiateFunc(FunctionInstantiator fInstantiaor)
             throws IOException {
+        super.instantiateFunc(fInstantiaor);
         for (EvalSpec spec: specs)
-            spec.instantiateFunc(fInstantiaor);
-        
+            spec.instantiateFunc(fInstantiaor);   
     }
 
     public List<EvalSpec> getSpecs() {

Modified: incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpec.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpec.java?rev=654172&r1=654171&r2=654172&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpec.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpec.java Wed May  7 
10:12:11 2008
@@ -199,17 +199,7 @@
      * @return
      */
     public Comparator<Tuple> getComparator() {
-        if (comparator != null)
-            return comparator;
-        else
-        {
-            comparator = new Comparator<Tuple>() {
-                public int compare(Tuple t1, Tuple t2) {
-                    return simpleEval(t1).compareTo(simpleEval(t2));
-                }
-            };
-            return comparator;
-        }
+        return comparator;
     }
     
     public void setFlatten(boolean isFlattened){

Modified: incubator/pig/trunk/src/org/apache/pig/impl/eval/FilterSpec.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/eval/FilterSpec.java?rev=654172&r1=654171&r2=654172&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/eval/FilterSpec.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/eval/FilterSpec.java Wed May  7 
10:12:11 2008
@@ -86,7 +86,8 @@
     @Override
     public void instantiateFunc(FunctionInstantiator instantiaor)
             throws IOException {
-        cond.instantiateFunc(instantiaor);        
+        super.instantiateFunc(instantiaor);
+        cond.instantiateFunc(instantiaor);
     }
 
     @Override

Modified: incubator/pig/trunk/src/org/apache/pig/impl/eval/FuncEvalSpec.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/eval/FuncEvalSpec.java?rev=654172&r1=654171&r2=654172&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/eval/FuncEvalSpec.java 
(original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/eval/FuncEvalSpec.java Wed May  
7 10:12:11 2008
@@ -56,6 +56,7 @@
     
     @Override
     public void instantiateFunc(FunctionInstantiator instantiaor) throws 
IOException{
+        super.instantiateFunc(instantiaor);
         if(instantiaor != null) {
             func = (EvalFunc) instantiaor.instantiateFuncFromAlias(funcName);
         }

Modified: incubator/pig/trunk/src/org/apache/pig/impl/eval/GenerateSpec.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/eval/GenerateSpec.java?rev=654172&r1=654171&r2=654172&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/eval/GenerateSpec.java 
(original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/eval/GenerateSpec.java Wed May  
7 10:12:11 2008
@@ -352,6 +352,7 @@
     @Override
     public void instantiateFunc(FunctionInstantiator instantiaor)
             throws IOException {
+        super.instantiateFunc(instantiaor);
         for (EvalSpec es: specs)
             es.instantiateFunc(instantiaor);        
     }

Modified: incubator/pig/trunk/src/org/apache/pig/impl/eval/SortDistinctSpec.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/eval/SortDistinctSpec.java?rev=654172&r1=654171&r2=654172&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/eval/SortDistinctSpec.java 
(original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/eval/SortDistinctSpec.java Wed 
May  7 10:12:11 2008
@@ -120,6 +120,7 @@
     @Override
     public void instantiateFunc(FunctionInstantiator instantiaor)
             throws IOException {
+        super.instantiateFunc(instantiaor);
         if (sortSpec!=null)
             sortSpec.instantiateFunc(instantiaor);        
     }


Reply via email to