Repository: systemml Updated Branches: refs/heads/master 9e6715daf -> 31952e47d
[MINOR] Fix mlcontext missing explain header and statistics reset This patch fixes two minor issues of the new mlcontext. First, it adds the missing explain header of available memory budgets and degree of parallelism by calling a new primitive that prepares the explain output (called for consistency from dmlscript and mlcontext). This also includes a slight modification of runtime explain which now avoids an unnecessary pass over the program. Second, it adds reset calls for the number of executed jobs and cache statistics which would otherwise accumulate over multiple runs of dml scripts in the same process. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/31952e47 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/31952e47 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/31952e47 Branch: refs/heads/master Commit: 31952e47d130ada3aab5dfb8dcc058b098ece90c Parents: 9e6715d Author: Matthias Boehm <[email protected]> Authored: Fri Jun 30 17:27:39 2017 -0700 Committer: Matthias Boehm <[email protected]> Committed: Fri Jun 30 18:05:56 2017 -0700 ---------------------------------------------------------------------- .../java/org/apache/sysml/api/DMLScript.java | 8 ++--- .../sysml/api/mlcontext/ScriptExecutor.java | 12 +++++-- .../java/org/apache/sysml/utils/Explain.java | 34 ++++++++++++++++---- .../mlcontext/MLContextParforDatasetTest.java | 5 +++ 4 files changed, 44 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/31952e47/src/main/java/org/apache/sysml/api/DMLScript.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java index 6a9077a..6dd7e89 100644 --- a/src/main/java/org/apache/sysml/api/DMLScript.java +++ b/src/main/java/org/apache/sysml/api/DMLScript.java @@ -741,12 +741,8 @@ public class DMLScript Statistics.resetNoOfCompiledJobs( counts.numJobs ); //explain plan of program (hops or runtime) - if( EXPLAIN != ExplainType.NONE ) { - LOG.info("EXPLAIN ("+EXPLAIN.toString()+"):\n" - + Explain.explainMemoryBudget(counts)+"\n" - + Explain.explainDegreeOfParallelism(counts) - + Explain.explain(prog, rtprog, EXPLAIN)); - } + if( EXPLAIN != ExplainType.NONE ) + LOG.info(Explain.display(prog, rtprog, EXPLAIN, counts)); Statistics.stopCompileTimer(); http://git-wip-us.apache.org/repos/asf/systemml/blob/31952e47/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java index 0035350..1a6a8ed 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java @@ -47,6 +47,7 @@ import org.apache.sysml.parser.ParserWrapper; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.controlprogram.LocalVariableMap; import org.apache.sysml.runtime.controlprogram.Program; +import org.apache.sysml.runtime.controlprogram.caching.CacheStatistics; import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory; import org.apache.sysml.utils.Explain; @@ -179,8 +180,10 @@ public class ScriptExecutor { return; try { - ExplainType explainType = (explainLevel != null) ? explainLevel.getExplainType() : ExplainType.RUNTIME; - System.out.println(Explain.explain(dmlProgram, runtimeProgram, explainType)); + ExplainType explainType = (explainLevel != null) ? + explainLevel.getExplainType() : ExplainType.RUNTIME; + System.out.println(Explain.display( + dmlProgram, runtimeProgram, explainType, null)); } catch (Exception e) { throw new MLContextException("Exception occurred while explaining dml program", e); } @@ -333,7 +336,10 @@ public class ScriptExecutor { // Set global variable indicating the script type DMLScript.SCRIPT_TYPE = script.getScriptType(); setGlobalFlags(); - if (statistics) { + //reset all relevant summary statistics + Statistics.resetNoOfExecutedJobs(); + if( statistics ) { + CacheStatistics.reset(); Statistics.reset(); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/31952e47/src/main/java/org/apache/sysml/utils/Explain.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/Explain.java b/src/main/java/org/apache/sysml/utils/Explain.java index 6451396..bd9d6aa 100644 --- a/src/main/java/org/apache/sysml/utils/Explain.java +++ b/src/main/java/org/apache/sysml/utils/Explain.java @@ -105,6 +105,18 @@ public class Explain ////////////// // public explain interface + public static String display(DMLProgram prog, Program rtprog, ExplainType type, ExplainCounts counts) + throws HopsException, DMLRuntimeException, LanguageException { + if( counts == null ) + counts = countDistributedOperations(rtprog); + + //explain plan of program (hops or runtime) + return "# EXPLAIN ("+type.name()+"):\n" + + Explain.explainMemoryBudget(counts)+"\n" + + Explain.explainDegreeOfParallelism(counts) + + Explain.explain(prog, rtprog, type, counts); + } + public static String explainMemoryBudget() { return explainMemoryBudget(new ExplainCounts()); } @@ -187,6 +199,11 @@ public class Explain } public static String explain(DMLProgram prog, Program rtprog, ExplainType type) + throws HopsException, DMLRuntimeException, LanguageException { + return explain(prog, rtprog, type); + } + + public static String explain(DMLProgram prog, Program rtprog, ExplainType type, ExplainCounts counts) throws HopsException, DMLRuntimeException, LanguageException { //dispatch to individual explain utils @@ -198,7 +215,7 @@ public class Explain //explain runtime program case RUNTIME: case RECOMPILE_RUNTIME: - return explain(rtprog); + return explain(rtprog, counts); case NONE: //do nothing } @@ -250,13 +267,19 @@ public class Explain return sb.toString(); } - public static String explain( Program rtprog ) + public static String explain( Program rtprog ) throws HopsException { + return explain(rtprog, null); + } + + public static String explain( Program rtprog, ExplainCounts counts ) throws HopsException { //counts number of instructions boolean sparkExec = OptimizerUtils.isSparkExecutionMode(); - ExplainCounts counts = new ExplainCounts(); - countCompiledInstructions(rtprog, counts, !sparkExec, true, sparkExec); + if( counts == null ) { + counts = new ExplainCounts(); + countCompiledInstructions(rtprog, counts, !sparkExec, true, sparkExec); + } StringBuilder sb = new StringBuilder(); @@ -426,8 +449,7 @@ public class Explain * @param rtprog runtime program * @return counts */ - public static ExplainCounts countDistributedOperations( Program rtprog ) - { + public static ExplainCounts countDistributedOperations( Program rtprog ) { ExplainCounts counts = new ExplainCounts(); if( OptimizerUtils.isSparkExecutionMode() ) Explain.countCompiledInstructions(rtprog, counts, false, true, true); http://git-wip-us.apache.org/repos/asf/systemml/blob/31952e47/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextParforDatasetTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextParforDatasetTest.java b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextParforDatasetTest.java index 36e7990..68b1373 100644 --- a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextParforDatasetTest.java +++ b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextParforDatasetTest.java @@ -32,6 +32,7 @@ import org.apache.sysml.api.mlcontext.MLResults; import org.apache.sysml.api.mlcontext.MatrixFormat; import org.apache.sysml.api.mlcontext.MatrixMetadata; import org.apache.sysml.api.mlcontext.Script; +import org.apache.sysml.api.mlcontext.MLContext.ExplainLevel; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext; import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer; @@ -152,6 +153,10 @@ public class MLContextParforDatasetTest extends AutomatedTestBase + "r = sum(v);"; String s = multiInputs ? s2 : s1; + ml.setExplain(true); + ml.setExplainLevel(ExplainLevel.RUNTIME); + ml.setStatistics(true); + Script script = dml(s).in("X", df, mm).out("r"); MLResults results = ml.execute(script);
