Author: gates Date: Fri Feb 29 20:13:45 2008 New Revision: 632546 URL: http://svn.apache.org/viewvc?rev=632546&view=rev Log: PIG-113 Make explain output more understandable.
Added: incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpecTreePrinter.java incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOTreePrinter.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POTreePrinter.java Modified: incubator/pig/trunk/CHANGES.txt incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapRedPhysicalPlan.java incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalPhysicalPlan.java incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java Modified: incubator/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=632546&r1=632545&r2=632546&view=diff ============================================================================== --- incubator/pig/trunk/CHANGES.txt (original) +++ incubator/pig/trunk/CHANGES.txt Fri Feb 29 20:13:45 2008 @@ -147,3 +147,5 @@ gates). PIG-13: adding version to the system (joa23 via olgan) + + PIG-113: Make explain output more understandable (pi_song via gates) Modified: incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapRedPhysicalPlan.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapRedPhysicalPlan.java?rev=632546&r1=632545&r2=632546&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapRedPhysicalPlan.java (original) +++ incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/MapRedPhysicalPlan.java Fri Feb 29 20:13:45 2008 @@ -10,6 +10,7 @@ import org.apache.pig.backend.executionengine.ExecPhysicalPlan; import org.apache.pig.impl.physicalLayer.PhysicalOperator; import org.apache.pig.impl.physicalLayer.POPrinter; +import org.apache.pig.impl.physicalLayer.POTreePrinter; import org.apache.pig.impl.physicalLayer.POVisitor; import org.apache.pig.impl.logicalLayer.OperatorKey; @@ -36,8 +37,7 @@ } public void explain(OutputStream out) { - POVisitor lprinter = new POPrinter(opTable, new PrintStream(out)); - + POVisitor lprinter = new POTreePrinter(opTable, new PrintStream(out)); ((PhysicalOperator)opTable.get(root)).visit(lprinter); } Modified: incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalPhysicalPlan.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalPhysicalPlan.java?rev=632546&r1=632545&r2=632546&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalPhysicalPlan.java (original) +++ incubator/pig/trunk/src/org/apache/pig/backend/local/executionengine/LocalPhysicalPlan.java Fri Feb 29 20:13:45 2008 @@ -8,6 +8,7 @@ import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.backend.executionengine.ExecPhysicalOperator; import org.apache.pig.backend.executionengine.ExecPhysicalPlan; +import org.apache.pig.impl.physicalLayer.POTreePrinter; import org.apache.pig.impl.physicalLayer.PhysicalOperator; import org.apache.pig.impl.physicalLayer.POVisitor; import org.apache.pig.impl.physicalLayer.POPrinter; @@ -35,8 +36,7 @@ } public void explain(OutputStream out) { - POVisitor lprinter = new POPrinter(opTable, new PrintStream(out)); - + POVisitor lprinter = new POTreePrinter(opTable, new PrintStream(out)); ((PhysicalOperator)opTable.get(root)).visit(lprinter); } Added: incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpecTreePrinter.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpecTreePrinter.java?rev=632546&view=auto ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpecTreePrinter.java (added) +++ incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpecTreePrinter.java Fri Feb 29 20:13:45 2008 @@ -0,0 +1,124 @@ +package org.apache.pig.impl.eval; + +import java.io.PrintStream; +import java.util.Iterator; + +import org.apache.pig.impl.eval.cond.AndCond; +import org.apache.pig.impl.eval.cond.CompCond; +import org.apache.pig.impl.eval.cond.FalseCond; +import org.apache.pig.impl.eval.cond.FuncCond; +import org.apache.pig.impl.eval.cond.NotCond; +import org.apache.pig.impl.eval.cond.OrCond; +import org.apache.pig.impl.eval.cond.RegexpCond; +import org.apache.pig.impl.eval.cond.TrueCond; + +public class EvalSpecTreePrinter extends EvalSpecVisitor { + private PrintStream mStream = null; + + public EvalSpecTreePrinter(PrintStream ps) { + mStream = ps; + } + + @Override + public void visitFilter(FilterSpec f) { + mStream.print("Filter: "); + if (f.cond instanceof AndCond) mStream.print(" AND "); + else if (f.cond instanceof CompCond) mStream.print(" COMP "); + else if (f.cond instanceof FalseCond) mStream.print(" FALSE "); + else if (f.cond instanceof FuncCond) mStream.print(" FUNC "); + else if (f.cond instanceof NotCond) mStream.print(" NOT "); + else if (f.cond instanceof OrCond) mStream.print(" OR "); + else if (f.cond instanceof RegexpCond) mStream.print(" REGEXP "); + else if (f.cond instanceof TrueCond) mStream.print(" TRUE "); + else throw new AssertionError(" Unknown Cond "); + } + + @Override + public void visitSortDistinct(SortDistinctSpec sd) { + mStream.print("Sort(") ; + if (sd.distinct()) { + mStream.print("Distinct(") ; + } + sd.getSortSpec().visit(this); + if (sd.distinct()) { + mStream.print(")") ; + } + mStream.print(")") ; + } + + @Override + public void visitGenerate(GenerateSpec g) { + mStream.print("Generate("); + Iterator<EvalSpec> i = g.getSpecs().iterator(); + boolean isFirst = true ; + while (i.hasNext()) { + if (isFirst) { + isFirst = false ; + } + else { + mStream.print(",") ; + } + i.next().visit(this); + } + mStream.print(")") ; + } + + @Override + public void visitMapLookup(MapLookupSpec ml) { + mStream.print("MapLookup(key=" + ml.key() + ")"); + } + + @Override + public void visitConst(ConstSpec c) { + mStream.print("Const(" + c.value() + ")"); + } + + @Override + public void visitProject(ProjectSpec p) { + mStream.print("Project("); + Iterator<Integer> i = p.getCols().iterator(); + while (i.hasNext()) { + mStream.print(i.next().intValue()); + if (i.hasNext()) mStream.print(","); + } + mStream.print(")"); + } + + @Override + public void visitStar(StarSpec s) { + mStream.print("*"); + } + + @Override + public void visitFuncEval(FuncEvalSpec fe) { + mStream.print("FuncEval(" + fe.getFuncName() + "("); + fe.getArgs().visit(this); + mStream.print("))") ; + } + + @Override + public void visitCompositeEval(CompositeEvalSpec ce) { + mStream.print("Composite("); + Iterator<EvalSpec> i = ce.getSpecs().iterator(); + boolean isFirst = true ; + while (i.hasNext()) { + if (isFirst) { + isFirst = false ; + } + else { + mStream.print(",") ; + } + i.next().visit(this); + } + mStream.print(")") ; + } + + @Override + public void visitBinCond(BinCondSpec bc) { + mStream.print("BinCond(True=>"); + bc.ifTrue().visit(this); + mStream.print (",False=>"); + bc.ifFalse().visit(this); + mStream.print (")"); + } +} Added: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOTreePrinter.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOTreePrinter.java?rev=632546&view=auto ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOTreePrinter.java (added) +++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOTreePrinter.java Fri Feb 29 20:13:45 2008 @@ -0,0 +1,151 @@ +package org.apache.pig.impl.logicalLayer; + +import java.io.PrintStream; +import java.util.Iterator; +import java.util.List; +import org.apache.pig.impl.logicalLayer.schema.Schema; +import org.apache.pig.impl.eval.cond.Cond; + +import org.apache.pig.impl.eval.EvalSpec; +import org.apache.pig.impl.eval.EvalSpecPrinter; + +public class LOTreePrinter extends LOVisitor { + + private static final String LINE_START_SYMBOL = "|---" ; + private static final String IDENT_SYMBOL = " " ; + + private int currentIdent = 0 ; + + private PrintStream mStream = null; + + public LOTreePrinter(PrintStream ps) { + mStream = ps; + } + + public void increaseIdent() { + currentIdent++ ; + } + + public void decreaseIdent() { + currentIdent-- ; + if (currentIdent <0) + { + throw new RuntimeException("Invalid LOTreePrinter state. currentIdent < 0") ; + } + } + + @Override + public void visitCogroup(LOCogroup g) { + printLineHeader(g, false) ; + mStream.print(" ( " + g.arguments() + " ) ") ; + mStream.println() ; + depthFirstSearchVisit(g) ; + } + + @Override + public void visitEval(LOEval g) { + printLineHeader(g, false) ; + mStream.print(" ( " + g.arguments() + " ) ") ; + mStream.println() ; + depthFirstSearchVisit(g) ; + } + + @Override + public void visitUnion(LOUnion g) { + printLineHeader(g) ; + depthFirstSearchVisit(g) ; + } + + + @Override + public void visitLoad(LOLoad g) { + printLineHeader(g, false) ; + mStream.print(" ( file = " + g.getInputFileSpec().getFileName() ) ; + + if (g.outputSchema().getFields().size() > 0) + { + mStream.print(" AS ") ; + boolean isFirst = true ; + for(Schema schema: g.outputSchema().getFields() ) { + if (isFirst) { + isFirst = false ; + } + else { + mStream.print(",") ; + } + mStream.print(schema.getAlias()) ; + } + } + + mStream.println(" )") ; + depthFirstSearchVisit(g) ; + } + + @Override + public void visitSort(LOSort g) { + printLineHeader(g, false) ; + mStream.print(" ( BY " + g.arguments() + " ) ") ; + mStream.println() ; + depthFirstSearchVisit(g) ; + } + + @Override + public void visitSplit(LOSplit g) { + printLineHeader(g, false) ; + mStream.print(" ( ") ; + boolean isFirst = true ; + for(Cond cond:g.getConditions()) { + if (isFirst) { + isFirst = false ; + } + else { + mStream.print(",") ; + } + mStream.print(cond) ; + } + mStream.println(" ) ") ; + depthFirstSearchVisit(g) ; + } + + @Override + public void visitSplitOutput(LOSplitOutput g) { + printLineHeader(g, false) ; + mStream.print(" ( " + g.arguments() + " ) ") ; + mStream.println() ; + depthFirstSearchVisit(g) ; + } + + @Override + public void visitStore(LOStore g) { + printLineHeader(g) ; + depthFirstSearchVisit(g) ; + } + + private void printLineHeader(LogicalOperator g) { + printLineHeader(g, true) ; + } + + private void printLineHeader(LogicalOperator g, boolean appendNewLine) { + for(int i=0;i<currentIdent;i++) { + mStream.print(IDENT_SYMBOL) ; + } + mStream.print(LINE_START_SYMBOL) ; + mStream.print(g.getClass().getSimpleName()) ; + + if (appendNewLine) { + mStream.println(); + } + } + + private void depthFirstSearchVisit(LogicalOperator lo) { + List<OperatorKey> inputs = lo.getInputs(); + Iterator<OperatorKey> i = inputs.iterator(); + + this.increaseIdent() ; + while (i.hasNext()) { + LogicalOperator input = lo.getOpTable().get(i.next()); + input.visit(this); + } + this.decreaseIdent() ; + } +} Modified: incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java?rev=632546&r1=632545&r2=632546&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java (original) +++ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java Fri Feb 29 20:13:45 2008 @@ -103,7 +103,7 @@ } public void explain(OutputStream out) { - LOVisitor lprinter = new LOPrinter(new PrintStream(out)); + LOVisitor lprinter = new LOTreePrinter(new PrintStream(out)); opTable.get(root).visit(lprinter); } Added: incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POTreePrinter.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POTreePrinter.java?rev=632546&view=auto ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POTreePrinter.java (added) +++ incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POTreePrinter.java Fri Feb 29 20:13:45 2008 @@ -0,0 +1,191 @@ +package org.apache.pig.impl.physicalLayer; + +import java.io.PrintStream; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.pig.backend.executionengine.ExecPhysicalOperator; +import org.apache.pig.backend.hadoop.executionengine.POMapreduce; +import org.apache.pig.backend.local.executionengine.POCogroup; +import org.apache.pig.backend.local.executionengine.POEval; +import org.apache.pig.backend.local.executionengine.POLoad; +import org.apache.pig.backend.local.executionengine.POSort; +import org.apache.pig.backend.local.executionengine.POSplit; +import org.apache.pig.backend.local.executionengine.POStore; +import org.apache.pig.backend.local.executionengine.POUnion; +import org.apache.pig.impl.eval.EvalSpec; +import org.apache.pig.impl.eval.EvalSpecTreePrinter; +import org.apache.pig.impl.io.FileSpec; +import org.apache.pig.impl.logicalLayer.OperatorKey; + +public class POTreePrinter extends POVisitor { + + private static final String HEADER_START_SYMBOL = "|---" ; + private static final String CONTENT_START_SYMBOL = " " ; + private static final String IDENT_SYMBOL = " " ; + + private int currentIdent = 0 ; + + private PrintStream mStream = null; + + public POTreePrinter(Map<OperatorKey, ExecPhysicalOperator> opTable, + PrintStream ps) { + super(opTable); + mStream = ps; + } + + public void increaseIdent() { + currentIdent++ ; + } + + public void decreaseIdent() { + currentIdent-- ; + if (currentIdent <0) + { + throw new RuntimeException("Invalid POTreePrinter state. currentIdent < 0") ; + } + } + + @Override + public void visitMapreduce(POMapreduce p) { + printLineHeader(p) ; + + // partition function + if (p.partitionFunction != null) { + adjustContentIdent() ; + mStream.println("Partition Function: " + p.partitionFunction.getName()); + mStream.println() ; + } + + // map line + adjustContentIdent() ; + mStream.print("Map : ") ; + visitSpecs(p.toMap); + mStream.println() ; + + // combine line + if (p.toCombine != null) { + adjustContentIdent() ; + mStream.print("Combine : ") ; + p.toCombine.visit(new EvalSpecTreePrinter(mStream)); + mStream.println() ; + } + + // reduce line + if (p.toReduce != null) { + adjustContentIdent() ; + mStream.print("Reduce : ") ; + p.toReduce.visit(new EvalSpecTreePrinter(mStream)); + mStream.println() ; + } + + // grouping line + if (p.groupFuncs != null) { + adjustContentIdent() ; + mStream.print("Grouping : ") ; + visitSpecs(p.groupFuncs); + mStream.println() ; + } + + // input files line + adjustContentIdent() ; + mStream.print("Input File(s) : ") ; + Iterator<FileSpec> i = p.inputFileSpecs.iterator(); + while (i.hasNext()) { + mStream.print(i.next().getFileName()); + if (i.hasNext()) mStream.print(", "); + } + mStream.println() ; + + depthFirstSearchVisit(p) ; + } + + @Override + public void visitLoad(POLoad p) { + printLineHeader(p) ; + depthFirstSearchVisit(p) ; + } + + @Override + public void visitSort(POSort p) { + printLineHeader(p) ; + depthFirstSearchVisit(p) ; + } + + @Override + public void visitStore(POStore p) { + printLineHeader(p) ; + depthFirstSearchVisit(p) ; + } + + @Override + public void visitCogroup(POCogroup p) { + printLineHeader(p) ; + depthFirstSearchVisit(p) ; + } + + @Override + public void visitEval(POEval p) { + printLineHeader(p) ; + depthFirstSearchVisit(p) ; + } + + @Override + public void visitSplit(POSplit p) { + printLineHeader(p) ; + depthFirstSearchVisit(p) ; + } + + @Override + public void visitUnion(POUnion p) { + printLineHeader(p) ; + depthFirstSearchVisit(p) ; + } + + private void printLineHeader(PhysicalOperator g) { + printLineHeader(g, true) ; + } + + private void printLineHeader(PhysicalOperator g, boolean appendNewLine) { + for(int i=0;i<currentIdent;i++) { + mStream.print(IDENT_SYMBOL) ; + } + mStream.print(HEADER_START_SYMBOL) ; + mStream.print(g.getClass().getSimpleName()) ; + + if (appendNewLine) { + mStream.println(); + } + } + + private void adjustContentIdent() { + for(int i=0;i<currentIdent;i++) { + mStream.print(IDENT_SYMBOL) ; + } + mStream.print(CONTENT_START_SYMBOL) ; + } + + private void depthFirstSearchVisit(PhysicalOperator po) { + this.increaseIdent() ; + for(OperatorKey inputKey : po.inputs) { + PhysicalOperator input = null ; + // since the only sub-type of ExecPhysicalOperator is PhysicalOperator + // it is legal to convert this way + input = (PhysicalOperator) mOpTable.get(inputKey) ; + if (input==null) { + throw new RuntimeException("Invalid OpKey table found while reading in POTreePrinter") ; + } + input.visit(this); + } + this.decreaseIdent() ; + } + + private void visitSpecs(List<EvalSpec> specs) { + Iterator<EvalSpec> j = specs.iterator(); + while (j.hasNext()) { + j.next().visit(new EvalSpecTreePrinter(mStream)); + } + } + +}