[SYSTEMML-1905] Extended codegen plan enumeration statistics (all plans) This patch extends the codegen plan enumeration statistics by the number of total plans without partitioning by connected components, i.e., the sum of 2^(|M|) per DAG.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/3acd9418 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/3acd9418 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/3acd9418 Branch: refs/heads/master Commit: 3acd94186b5e6e2fdf12823e7932bf56027511c6 Parents: 808a8f4 Author: Matthias Boehm <mboe...@gmail.com> Authored: Tue Sep 12 19:10:58 2017 -0700 Committer: Matthias Boehm <mboe...@gmail.com> Committed: Tue Sep 12 20:13:18 2017 -0700 ---------------------------------------------------------------------- .../hops/codegen/opt/PlanSelectionFuseCostBased.java | 9 ++++++++- .../hops/codegen/opt/PlanSelectionFuseCostBasedV2.java | 8 +++++++- src/main/java/org/apache/sysml/utils/Statistics.java | 12 ++++++++++-- 3 files changed, 25 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/3acd9418/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBased.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBased.java b/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBased.java index 4fcef6e..c230505 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBased.java +++ b/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBased.java @@ -57,6 +57,7 @@ import org.apache.sysml.hops.codegen.template.TemplateBase.TemplateType; import org.apache.sysml.hops.rewrite.HopRewriteUtils; import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer; import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence; +import org.apache.sysml.runtime.util.UtilFunctions; import org.apache.sysml.utils.Statistics; /** @@ -87,12 +88,14 @@ public class PlanSelectionFuseCostBased extends PlanSelection Collection<PlanPartition> parts = PlanAnalyzer.analyzePlanPartitions(memo, roots, false); //step 2: optimize individual plan partitions + int sumMatPoints = 0; for( PlanPartition part : parts ) { //create composite templates (within the partition) createAndAddMultiAggPlans(memo, part.getPartition(), part.getRoots()); //plan enumeration and plan selection selectPlans(memo, part.getPartition(), part.getRoots(), part.getMatPoints()); + sumMatPoints += part.getMatPoints().size(); } //step 3: add composite templates (across partitions) @@ -101,6 +104,10 @@ public class PlanSelectionFuseCostBased extends PlanSelection //take all distinct best plans for( Entry<Long, List<MemoTableEntry>> e : getBestPlans().entrySet() ) memo.setDistinct(e.getKey(), e.getValue()); + + //maintain statistics + if( DMLScript.STATISTICS ) + Statistics.incrementCodegenEnumAll(UtilFunctions.pow(2, sumMatPoints)); } //within-partition multi-agg templates @@ -389,7 +396,7 @@ public class PlanSelectionFuseCostBased extends PlanSelection } if( DMLScript.STATISTICS ) { - Statistics.incrementCodegenEnumAll(len); + Statistics.incrementCodegenEnumAllP(len); Statistics.incrementCodegenEnumEval(len); } http://git-wip-us.apache.org/repos/asf/systemml/blob/3acd9418/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java b/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java index a8f7365..1bf42f6 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java +++ b/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java @@ -107,12 +107,14 @@ public class PlanSelectionFuseCostBasedV2 extends PlanSelection Collection<PlanPartition> parts = PlanAnalyzer.analyzePlanPartitions(memo, roots, true); //step 2: optimize individual plan partitions + int sumMatPoints = 0; for( PlanPartition part : parts ) { //create composite templates (within the partition) createAndAddMultiAggPlans(memo, part.getPartition(), part.getRoots()); //plan enumeration and plan selection selectPlans(memo, part); + sumMatPoints += part.getMatPointsExt().length; } //step 3: add composite templates (across partitions) @@ -121,6 +123,10 @@ public class PlanSelectionFuseCostBasedV2 extends PlanSelection //take all distinct best plans for( Entry<Long, List<MemoTableEntry>> e : getBestPlans().entrySet() ) memo.setDistinct(e.getKey(), e.getValue()); + + //maintain statistics + if( DMLScript.STATISTICS ) + Statistics.incrementCodegenEnumAll(UtilFunctions.pow(2, sumMatPoints)); } private void selectPlans(CPlanMemoTable memo, PlanPartition part) @@ -257,7 +263,7 @@ public class PlanSelectionFuseCostBasedV2 extends PlanSelection } if( DMLScript.STATISTICS ) { - Statistics.incrementCodegenEnumAll((rgraph!=null)?len:0); + Statistics.incrementCodegenEnumAllP((rgraph!=null)?len:0); Statistics.incrementCodegenEnumEval(numEvalPlans); Statistics.incrementCodegenEnumEvalP(numEvalPartPlans); } http://git-wip-us.apache.org/repos/asf/systemml/blob/3acd9418/src/main/java/org/apache/sysml/utils/Statistics.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/Statistics.java b/src/main/java/org/apache/sysml/utils/Statistics.java index bab668f..109eec2 100644 --- a/src/main/java/org/apache/sysml/utils/Statistics.java +++ b/src/main/java/org/apache/sysml/utils/Statistics.java @@ -79,6 +79,7 @@ public class Statistics private static final LongAdder codegenCPlanCompile = new LongAdder(); //count private static final LongAdder codegenClassCompile = new LongAdder(); //count private static final LongAdder codegenEnumAll = new LongAdder(); //count + private static final LongAdder codegenEnumAllP = new LongAdder(); //count private static final LongAdder codegenEnumEval = new LongAdder(); //count private static final LongAdder codegenEnumEvalP = new LongAdder(); //count private static final LongAdder codegenPlanCacheHits = new LongAdder(); //count @@ -262,6 +263,9 @@ public class Statistics public static void incrementCodegenEnumAll(long delta) { codegenEnumAll.add(delta); } + public static void incrementCodegenEnumAllP(long delta) { + codegenEnumAllP.add(delta); + } public static void incrementCodegenEnumEval(long delta) { codegenEnumEval.add(delta); } @@ -300,6 +304,9 @@ public class Statistics public static long getCodegenEnumAll() { return codegenEnumAll.longValue(); } + public static long getCodegenEnumAllP() { + return codegenEnumAllP.longValue(); + } public static long getCodegenEnumEval() { return codegenEnumEval.longValue(); } @@ -403,6 +410,7 @@ public class Statistics codegenCPlanCompile.reset(); codegenClassCompile.reset(); codegenEnumAll.reset(); + codegenEnumAllP.reset(); codegenEnumEval.reset(); codegenEnumEvalP.reset(); codegenCompileTime.reset(); @@ -795,8 +803,8 @@ public class Statistics if( ConfigurationManager.isCodegenEnabled() ) { sb.append("Codegen compile (DAG,CP,JC):\t" + getCodegenDAGCompile() + "/" + getCodegenCPlanCompile() + "/" + getCodegenClassCompile() + ".\n"); - sb.append("Codegen enum (All,Eval,EvalP):\t" + getCodegenEnumAll() + "/" - + getCodegenEnumEval() + "/" + getCodegenEnumEvalP() + ".\n"); + sb.append("Codegen enum (ALLt/p,EVALt/p):\t" + getCodegenEnumAll() + "/" + + getCodegenEnumAllP() + "/" + getCodegenEnumEval() + "/" + getCodegenEnumEvalP() + ".\n"); sb.append("Codegen compile times (DAG,JC):\t" + String.format("%.3f", (double)getCodegenCompileTime()/1000000000) + "/" + String.format("%.3f", (double)getCodegenClassCompileTime()/1000000000) + " sec.\n"); sb.append("Codegen plan cache hits:\t" + getCodegenPlanCacheHits() + "/" + getCodegenPlanCacheTotal() + ".\n");
