gates
Wed, 07 May 2008 10:12:43 -0700
Author: gates Date: Wed May 7 10:12:11 2008 New Revision: 654172 URL: http://svn.apache.org/viewvc?rev=654172&view=rev Log: PIG-202: Fix Order by so that user provided comparator func is used for quantile determination. Added: incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/InstantiateFuncCallerPOVisitor.java Modified: incubator/pig/trunk/CHANGES.txt incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapreducePlanCompiler.java incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java incubator/pig/trunk/src/org/apache/pig/impl/builtin/FindQuantiles.java incubator/pig/trunk/src/org/apache/pig/impl/eval/BinCondSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/CompositeEvalSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/FilterSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/FuncEvalSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/GenerateSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/SortDistinctSpec.java Modified: incubator/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=654172&r1=654171&r2=654172&view=diff ============================================================================== --- incubator/pig/trunk/CHANGES.txt (original) +++ incubator/pig/trunk/CHANGES.txt Wed May 7 10:12:11 2008 @@ -276,3 +276,5 @@ PIG-229: Proper error handling in case of deserializer failure PIG-219: Change unit tests to run both local and map reduce modes (kali via gates). + + PIG-202: Fix Order by so that user provided comparator func is used for quantile determination (kali via gates). Modified: incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapreducePlanCompiler.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapreducePlanCompiler.java?rev=654172&r1=654171&r2=654172&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapreducePlanCompiler.java (original) +++ incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapreducePlanCompiler.java Wed May 7 10:12:11 2008 @@ -353,7 +353,9 @@ //sort the first column of the cogroup output and feed it to the quantiles function EvalSpec sortedSampleSpec = new ProjectSpec(1); - sortedSampleSpec = sortedSampleSpec.addSpec(new SortDistinctSpec(false, new StarSpec())); + EvalSpec starSpec = new StarSpec(); + starSpec.setComparatorName(loSort.getSortSpec().getComparatorName()); + sortedSampleSpec = sortedSampleSpec.addSpec(new SortDistinctSpec(false, starSpec)); argsList.add(sortedSampleSpec); EvalSpec args = new GenerateSpec(argsList); Added: incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/InstantiateFuncCallerPOVisitor.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/InstantiateFuncCallerPOVisitor.java?rev=654172&view=auto ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/InstantiateFuncCallerPOVisitor.java (added) +++ incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/InstantiateFuncCallerPOVisitor.java Wed May 7 10:12:11 2008 @@ -0,0 +1,62 @@ +/** + * + */ +package org.apache.pig.backend.local.executionengine; + +import java.io.IOException; +import java.util.Map; + +import org.apache.pig.backend.executionengine.ExecPhysicalOperator; +import org.apache.pig.backend.hadoop.executionengine.POMapreduce; +import org.apache.pig.impl.FunctionInstantiator; +import org.apache.pig.impl.eval.EvalSpec; +import org.apache.pig.impl.logicalLayer.OperatorKey; +import org.apache.pig.impl.physicalLayer.POVisitor; + +/** + * POVisitor calling instantiateFunc on all EvalSpec members of visited nodes. + */ +public class InstantiateFuncCallerPOVisitor extends POVisitor { + + private FunctionInstantiator instantiator; + + protected InstantiateFuncCallerPOVisitor(FunctionInstantiator instantiator, + Map<OperatorKey, ExecPhysicalOperator> opTable) { + super(opTable); + this.instantiator = instantiator; + } + + private void callInstantiateFunc(EvalSpec spec) { + try { + spec.instantiateFunc(instantiator); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void visitCogroup(POCogroup g) { + super.visitCogroup(g); + for (EvalSpec es : g.specs) + callInstantiateFunc(es); + } + + @Override + public void visitMapreduce(POMapreduce mr) { + super.visitMapreduce(mr); + for (EvalSpec es : mr.groupFuncs) + callInstantiateFunc(es); + } + + @Override + public void visitSort(POSort s) { + super.visitSort(s); + callInstantiateFunc(s.sortSpec); + } + + @Override + public void visitEval(POEval e) { + super.visitEval(e); + callInstantiateFunc(e.spec); + } +} Modified: incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java?rev=654172&r1=654171&r2=654172&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java (original) +++ incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalExecutionEngine.java Wed May 7 10:12:11 2008 @@ -122,6 +122,7 @@ try { PhysicalOperator pp = (PhysicalOperator)physicalOpTable.get(plan.getRoot()); + pp.visit(new InstantiateFuncCallerPOVisitor(pigContext, physicalOpTable)); pp.open(); Tuple t; Modified: incubator/pig/trunk/src/org/apache/pig/impl/builtin/FindQuantiles.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/builtin/FindQuantiles.java?rev=654172&r1=654171&r2=654172&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/impl/builtin/FindQuantiles.java (original) +++ incubator/pig/trunk/src/org/apache/pig/impl/builtin/FindQuantiles.java Wed May 7 10:12:11 2008 @@ -39,15 +39,13 @@ long numSamples = samples.size(); - long toSkip = numSamples / numQuantiles; - - long i=0, nextQuantile = 0; + long i=0, written = 0; Iterator<Tuple> iter = samples.iterator(); while (iter.hasNext()){ Tuple t = iter.next(); - if (i==nextQuantile){ + if ((written + 1) * numSamples < i * (numQuantiles + 1)) { output.add(t); - nextQuantile+=toSkip+1; + written++; } i++; } Modified: incubator/pig/trunk/src/org/apache/pig/impl/eval/BinCondSpec.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/eval/BinCondSpec.java?rev=654172&r1=654171&r2=654172&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/impl/eval/BinCondSpec.java (original) +++ incubator/pig/trunk/src/org/apache/pig/impl/eval/BinCondSpec.java Wed May 7 10:12:11 2008 @@ -58,6 +58,7 @@ @Override public void instantiateFunc(FunctionInstantiator fInstantiaor) throws IOException{ + super.instantiateFunc(fInstantiaor); cond.instantiateFunc(fInstantiaor); ifTrue.instantiateFunc(fInstantiaor); ifFalse.instantiateFunc(fInstantiaor); Modified: incubator/pig/trunk/src/org/apache/pig/impl/eval/CompositeEvalSpec.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/eval/CompositeEvalSpec.java?rev=654172&r1=654171&r2=654172&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/impl/eval/CompositeEvalSpec.java (original) +++ incubator/pig/trunk/src/org/apache/pig/impl/eval/CompositeEvalSpec.java Wed May 7 10:12:11 2008 @@ -102,9 +102,9 @@ @Override public void instantiateFunc(FunctionInstantiator fInstantiaor) throws IOException { + super.instantiateFunc(fInstantiaor); for (EvalSpec spec: specs) - spec.instantiateFunc(fInstantiaor); - + spec.instantiateFunc(fInstantiaor); } public List<EvalSpec> getSpecs() { Modified: incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpec.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpec.java?rev=654172&r1=654171&r2=654172&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpec.java (original) +++ incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpec.java Wed May 7 10:12:11 2008 @@ -199,17 +199,7 @@ * @return */ public Comparator<Tuple> getComparator() { - if (comparator != null) - return comparator; - else - { - comparator = new Comparator<Tuple>() { - public int compare(Tuple t1, Tuple t2) { - return simpleEval(t1).compareTo(simpleEval(t2)); - } - }; - return comparator; - } + return comparator; } public void setFlatten(boolean isFlattened){ Modified: incubator/pig/trunk/src/org/apache/pig/impl/eval/FilterSpec.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/eval/FilterSpec.java?rev=654172&r1=654171&r2=654172&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/impl/eval/FilterSpec.java (original) +++ incubator/pig/trunk/src/org/apache/pig/impl/eval/FilterSpec.java Wed May 7 10:12:11 2008 @@ -86,7 +86,8 @@ @Override public void instantiateFunc(FunctionInstantiator instantiaor) throws IOException { - cond.instantiateFunc(instantiaor); + super.instantiateFunc(instantiaor); + cond.instantiateFunc(instantiaor); } @Override Modified: incubator/pig/trunk/src/org/apache/pig/impl/eval/FuncEvalSpec.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/eval/FuncEvalSpec.java?rev=654172&r1=654171&r2=654172&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/impl/eval/FuncEvalSpec.java (original) +++ incubator/pig/trunk/src/org/apache/pig/impl/eval/FuncEvalSpec.java Wed May 7 10:12:11 2008 @@ -56,6 +56,7 @@ @Override public void instantiateFunc(FunctionInstantiator instantiaor) throws IOException{ + super.instantiateFunc(instantiaor); if(instantiaor != null) { func = (EvalFunc) instantiaor.instantiateFuncFromAlias(funcName); } Modified: incubator/pig/trunk/src/org/apache/pig/impl/eval/GenerateSpec.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/eval/GenerateSpec.java?rev=654172&r1=654171&r2=654172&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/impl/eval/GenerateSpec.java (original) +++ incubator/pig/trunk/src/org/apache/pig/impl/eval/GenerateSpec.java Wed May 7 10:12:11 2008 @@ -352,6 +352,7 @@ @Override public void instantiateFunc(FunctionInstantiator instantiaor) throws IOException { + super.instantiateFunc(instantiaor); for (EvalSpec es: specs) es.instantiateFunc(instantiaor); } Modified: incubator/pig/trunk/src/org/apache/pig/impl/eval/SortDistinctSpec.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/eval/SortDistinctSpec.java?rev=654172&r1=654171&r2=654172&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/impl/eval/SortDistinctSpec.java (original) +++ incubator/pig/trunk/src/org/apache/pig/impl/eval/SortDistinctSpec.java Wed May 7 10:12:11 2008 @@ -120,6 +120,7 @@ @Override public void instantiateFunc(FunctionInstantiator instantiaor) throws IOException { + super.instantiateFunc(instantiaor); if (sortSpec!=null) sortSpec.instantiateFunc(instantiaor); }