Repository: incubator-systemml Updated Branches: refs/heads/master 4fff6f769 -> 0827a4638
[SYSTEMML-837] MLContext Explain options Closes #214. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/0827a463 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/0827a463 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/0827a463 Branch: refs/heads/master Commit: 0827a46385b08848ffa5556c1ac2f15e5c860585 Parents: 4fff6f7 Author: Deron Eriksson <[email protected]> Authored: Fri Aug 19 16:28:06 2016 -0700 Committer: Deron Eriksson <[email protected]> Committed: Fri Aug 19 16:28:06 2016 -0700 ---------------------------------------------------------------------- docs/spark-mlcontext-programming-guide.md | 66 ++++++++++++++++---- .../apache/sysml/api/mlcontext/MLContext.java | 61 ++++++++++++++++-- .../sysml/api/mlcontext/ScriptExecutor.java | 31 ++++++++- 3 files changed, 140 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0827a463/docs/spark-mlcontext-programming-guide.md ---------------------------------------------------------------------- diff --git a/docs/spark-mlcontext-programming-guide.md b/docs/spark-mlcontext-programming-guide.md index 71db1a4..d0e1f0f 100644 --- a/docs/spark-mlcontext-programming-guide.md +++ b/docs/spark-mlcontext-programming-guide.md @@ -1108,6 +1108,7 @@ minOut = min(Xin) maxOut = max(Xin) meanOut = mean(Xin) """ +val mm = new MatrixMetadata(numRows, numCols) val minMaxMeanScript = dml(minMaxMean).in("Xin", df, mm).out("minOut", "maxOut", "meanOut") val (min, max, mean) = ml.execute(minMaxMeanScript).getTuple[Double, Double, Double]("minOut", "maxOut", "meanOut") @@ -1131,6 +1132,9 @@ maxOut = max(Xin) meanOut = mean(Xin) " +scala> val mm = new MatrixMetadata(numRows, numCols) +mm: org.apache.sysml.api.mlcontext.MatrixMetadata = rows: 10000, columns: 1000, non-zeros: None, rows per block: None, columns per block: None + scala> val minMaxMeanScript = dml(minMaxMean).in("Xin", df, mm).out("minOut", "maxOut", "meanOut") minMaxMeanScript: org.apache.sysml.api.mlcontext.Script = Inputs: @@ -1147,17 +1151,57 @@ scala> val (min, max, mean) = ml.execute(minMaxMeanScript).getTuple[Double, Doub PROGRAM --MAIN PROGRAM ----GENERIC (lines 1-8) [recompile=false] -------(4959) PRead Xin [10000,1000,1000,1000,10000000] [0,0,76 -> 76MB] [chkpt] -------(4960) ua(minRC) (4959) [0,0,-1,-1,-1] [76,0,0 -> 76MB] -------(4968) PWrite minOut (4960) [0,0,-1,-1,-1] [0,0,0 -> 0MB] -------(4961) ua(maxRC) (4959) [0,0,-1,-1,-1] [76,0,0 -> 76MB] -------(4974) PWrite maxOut (4961) [0,0,-1,-1,-1] [0,0,0 -> 0MB] -------(4962) ua(meanRC) (4959) [0,0,-1,-1,-1] [76,0,0 -> 76MB] -------(4980) PWrite meanOut (4962) [0,0,-1,-1,-1] [0,0,0 -> 0MB] - -min: Double = 3.682402316407263E-8 -max: Double = 0.999999984664141 -mean: Double = 0.49997351913605814 +------(12) TRead Xin [10000,1000,1000,1000,10000000] [0,0,76 -> 76MB] [chkpt], CP +------(13) ua(minRC) (12) [0,0,-1,-1,-1] [76,0,0 -> 76MB], CP +------(21) TWrite minOut (13) [0,0,-1,-1,-1] [0,0,0 -> 0MB], CP +------(14) ua(maxRC) (12) [0,0,-1,-1,-1] [76,0,0 -> 76MB], CP +------(27) TWrite maxOut (14) [0,0,-1,-1,-1] [0,0,0 -> 0MB], CP +------(15) ua(meanRC) (12) [0,0,-1,-1,-1] [76,0,0 -> 76MB], CP +------(33) TWrite meanOut (15) [0,0,-1,-1,-1] [0,0,0 -> 0MB], CP + +min: Double = 5.16651366133658E-9 +max: Double = 0.9999999368927975 +mean: Double = 0.5001096515241128 + +{% endhighlight %} +</div> + +</div> + + +Different explain levels can be set. The explain levels are NONE, HOPS, RUNTIME, RECOMPILE_HOPS, and RECOMPILE_RUNTIME. + +<div class="codetabs"> + +<div data-lang="Scala" markdown="1"> +{% highlight scala %} +ml.setExplainLevel(MLContext.ExplainLevel.RUNTIME) +val (min, max, mean) = ml.execute(minMaxMeanScript).getTuple[Double, Double, Double]("minOut", "maxOut", "meanOut") +{% endhighlight %} +</div> + +<div data-lang="Spark Shell" markdown="1"> +{% highlight scala %} +scala> ml.setExplainLevel(MLContext.ExplainLevel.RUNTIME) + +scala> val (min, max, mean) = ml.execute(minMaxMeanScript).getTuple[Double, Double, Double]("minOut", "maxOut", "meanOut") + +PROGRAM ( size CP/SP = 9/0 ) +--MAIN PROGRAM +----GENERIC (lines 1-8) [recompile=false] +------CP uamin Xin.MATRIX.DOUBLE _Var8.SCALAR.DOUBLE 8 +------CP uamax Xin.MATRIX.DOUBLE _Var9.SCALAR.DOUBLE 8 +------CP uamean Xin.MATRIX.DOUBLE _Var10.SCALAR.DOUBLE 8 +------CP assignvar _Var8.SCALAR.DOUBLE.false minOut.SCALAR.DOUBLE +------CP assignvar _Var9.SCALAR.DOUBLE.false maxOut.SCALAR.DOUBLE +------CP assignvar _Var10.SCALAR.DOUBLE.false meanOut.SCALAR.DOUBLE +------CP rmvar _Var8 +------CP rmvar _Var9 +------CP rmvar _Var10 + +min: Double = 5.16651366133658E-9 +max: Double = 0.9999999368927975 +mean: Double = 0.5001096515241128 {% endhighlight %} </div> http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0827a463/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java index 85e2143..8f809f8 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java @@ -50,6 +50,7 @@ import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction; import org.apache.sysml.runtime.instructions.spark.functions.SparkListener; import org.apache.sysml.runtime.matrix.MatrixFormatMetaData; import org.apache.sysml.runtime.matrix.data.OutputInfo; +import org.apache.sysml.utils.Explain.ExplainType; /** * The MLContext API offers programmatic access to SystemML on Spark from @@ -99,10 +100,50 @@ public class MLContext { */ private boolean statistics = false; + /** + * The level and type of program explanation that should be displayed if + * explain is set to true. + */ + private ExplainLevel explainLevel = null; + private List<String> scriptHistoryStrings = new ArrayList<String>(); private Map<String, Script> scripts = new LinkedHashMap<String, Script>(); /** + * The different explain levels supported by SystemML. + * + */ + public enum ExplainLevel { + /** Explain disabled */ + NONE, + /** Explain program and HOPs */ + HOPS, + /** Explain runtime program */ + RUNTIME, + /** Explain HOPs, including recompile */ + RECOMPILE_HOPS, + /** Explain runtime program, including recompile */ + RECOMPILE_RUNTIME; + + public ExplainType getExplainType() { + switch (this) { + case NONE: + return ExplainType.NONE; + case HOPS: + return ExplainType.HOPS; + case RUNTIME: + return ExplainType.RUNTIME; + case RECOMPILE_HOPS: + return ExplainType.RECOMPILE_HOPS; + case RECOMPILE_RUNTIME: + return ExplainType.RECOMPILE_RUNTIME; + default: + return ExplainType.HOPS; + } + } + }; + + /** * Retrieve the currently active MLContext. This is used internally by * SystemML via MLContextProxy. * @@ -225,6 +266,7 @@ public class MLContext { public MLResults execute(Script script) { ScriptExecutor scriptExecutor = new ScriptExecutor(sparkMonitoringUtil); scriptExecutor.setExplain(explain); + scriptExecutor.setExplainLevel(explainLevel); scriptExecutor.setStatistics(statistics); return execute(script, scriptExecutor); } @@ -312,6 +354,17 @@ public class MLContext { } /** + * Set the level of program explanation that should be displayed if explain + * is set to true. + * + * @param explainLevel + * the level of program explanation + */ + public void setExplainLevel(ExplainLevel explainLevel) { + this.explainLevel = explainLevel; + } + + /** * Used internally by MLContextProxy. * */ @@ -503,13 +556,13 @@ public class MLContext { } public void close() { - //reset static status (refs to sc / mlcontext) + // reset static status (refs to sc / mlcontext) SparkExecutionContext.resetSparkContextStatic(); MLContextProxy.setActive(false); activeMLContext = null; - - //clear local status, but do not stop sc as it - //may be used or stopped externally + + // clear local status, but do not stop sc as it + // may be used or stopped externally clear(); resetConfig(); sc = null; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0827a463/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java index 4702af2..cd4797c 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java @@ -26,6 +26,7 @@ import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.jmlc.JMLCUtils; +import org.apache.sysml.api.mlcontext.MLContext.ExplainLevel; import org.apache.sysml.api.monitoring.SparkMonitoringUtil; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.conf.DMLConfig; @@ -48,6 +49,7 @@ import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory; import org.apache.sysml.utils.Explain; import org.apache.sysml.utils.Explain.ExplainCounts; +import org.apache.sysml.utils.Explain.ExplainType; import org.apache.sysml.utils.Statistics; /** @@ -114,6 +116,7 @@ public class ScriptExecutor { protected Script script; protected boolean explain = false; protected boolean statistics = false; + protected ExplainLevel explainLevel; /** * ScriptExecutor constructor. @@ -197,7 +200,12 @@ public class ScriptExecutor { protected void showExplanation() { if (explain) { try { - System.out.println(Explain.explain(dmlProgram)); + if (explainLevel == null) { + System.out.println(Explain.explain(dmlProgram)); + } else { + ExplainType explainType = explainLevel.getExplainType(); + System.out.println(Explain.explain(dmlProgram, runtimeProgram, explainType)); + } } catch (HopsException e) { throw new MLContextException("Exception occurred while explaining dml program", e); } catch (DMLRuntimeException e) { @@ -276,10 +284,10 @@ public class ScriptExecutor { * <li>{@link #validateScript()}</li> * <li>{@link #constructHops()}</li> * <li>{@link #rewriteHops()}</li> - * <li>{@link #showExplanation()}</li> * <li>{@link #rewritePersistentReadsAndWrites()}</li> * <li>{@link #constructLops()}</li> * <li>{@link #generateRuntimeProgram()}</li> + * <li>{@link #showExplanation()}</li> * <li>{@link #globalDataFlowOptimization()}</li> * <li>{@link #countCompiledMRJobsAndSparkInstructions()}</li> * <li>{@link #initializeCachingAndScratchSpace()}</li> @@ -304,10 +312,10 @@ public class ScriptExecutor { validateScript(); constructHops(); rewriteHops(); - showExplanation(); rewritePersistentReadsAndWrites(); constructLops(); generateRuntimeProgram(); + showExplanation(); globalDataFlowOptimization(); countCompiledMRJobsAndSparkInstructions(); initializeCachingAndScratchSpace(); @@ -621,4 +629,21 @@ public class ScriptExecutor { this.statistics = statistics; } + /** + * Set the level of program explanation that should be displayed if explain + * is set to true. + * + * @param explainLevel + * the level of program explanation + */ + public void setExplainLevel(ExplainLevel explainLevel) { + this.explainLevel = explainLevel; + if (explainLevel == null) { + DMLScript.EXPLAIN = ExplainType.NONE; + } else { + ExplainType explainType = explainLevel.getExplainType(); + DMLScript.EXPLAIN = explainType; + } + } + }
