Repository: incubator-systemml Updated Branches: refs/heads/master 2fbed1f1a -> a27663946
[SYSTEMML-1598] Output statistics via MLContext to stdout If ml.setStatistics(true), output statistics to standard output rather than to log4j. Closes #491. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/a2766394 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/a2766394 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/a2766394 Branch: refs/heads/master Commit: a276639460b6e994007b13befe07975d1606b678 Parents: 2fbed1f Author: Deron Eriksson <de...@us.ibm.com> Authored: Thu May 11 15:52:55 2017 -0700 Committer: Deron Eriksson <de...@us.ibm.com> Committed: Thu May 11 15:52:55 2017 -0700 ---------------------------------------------------------------------- .../apache/sysml/api/ScriptExecutorUtils.java | 79 +++++++++++++------- .../apache/sysml/api/mlcontext/MLContext.java | 9 +++ .../sysml/api/mlcontext/ScriptExecutor.java | 25 +++++-- 3 files changed, 82 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a2766394/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java b/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java index 543e218..0345c62 100644 --- a/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java +++ b/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java @@ -16,10 +16,11 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.sysml.api; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.apache.sysml.api.mlcontext.MLContext; +import org.apache.sysml.api.mlcontext.ScriptExecutor; import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.hops.codegen.SpoofCompiler; import org.apache.sysml.runtime.DMLRuntimeException; @@ -32,52 +33,78 @@ import org.apache.sysml.utils.GPUStatistics; import org.apache.sysml.utils.Statistics; public class ScriptExecutorUtils { - private static final Log LOG = LogFactory.getLog(ScriptExecutorUtils.class.getName()); - + /** * Execute the runtime program. This involves execution of the program * blocks that make up the runtime program and may involve dynamic * recompilation. * - * @param rtprog runtime program - * @param ec execution context - * @param dmlconf dml configuration - * @throws DMLRuntimeException if error occurs + * @param se + * script executor + * @throws DMLRuntimeException + * if exception occurs */ - public static void executeRuntimeProgram(Program rtprog, ExecutionContext ec, DMLConfig dmlconf) throws DMLRuntimeException { - // Whether extra statistics useful for developers and others interested in digging - // into performance problems are recorded and displayed + public static void executeRuntimeProgram(ScriptExecutor se) throws DMLRuntimeException { + Program prog = se.getRuntimeProgram(); + ExecutionContext ec = se.getExecutionContext(); + DMLConfig config = se.getConfig(); + executeRuntimeProgram(prog, ec, config); + } + + /** + * Execute the runtime program. This involves execution of the program + * blocks that make up the runtime program and may involve dynamic + * recompilation. + * + * @param rtprog + * runtime program + * @param ec + * execution context + * @param dmlconf + * dml configuration + * @throws DMLRuntimeException + * if error occurs + */ + public static void executeRuntimeProgram(Program rtprog, ExecutionContext ec, DMLConfig dmlconf) + throws DMLRuntimeException { + // Whether extra statistics useful for developers and others interested + // in digging into performance problems are recorded and displayed GPUStatistics.DISPLAY_STATISTICS = dmlconf.getBooleanValue(DMLConfig.EXTRA_GPU_STATS); LibMatrixDNN.DISPLAY_STATISTICS = dmlconf.getBooleanValue(DMLConfig.EXTRA_DNN_STATS); - // Sets the maximum number of GPUs per process, -1 for all available GPUs + // Sets the maximum number of GPUs per process, -1 for all available + // GPUs GPUContextPool.PER_PROCESS_MAX_GPUS = dmlconf.getIntValue(DMLConfig.MAX_GPUS_PER_PROCESS); Statistics.startRunTimer(); GPUContext gCtx = null; - try { - //run execute (w/ exception handling to ensure proper shutdown) - if (DMLScript.USE_ACCELERATOR && ec != null){ + try { + // run execute (w/ exception handling to ensure proper shutdown) + if (DMLScript.USE_ACCELERATOR && ec != null) { gCtx = GPUContextPool.getFromPool(); if (gCtx == null) { - throw new DMLRuntimeException("GPU : Could not create GPUContext, either no GPU or all GPUs currently in use"); + throw new DMLRuntimeException( + "GPU : Could not create GPUContext, either no GPU or all GPUs currently in use"); } gCtx.initializeThread(); ec.setGPUContext(gCtx); } - rtprog.execute( ec ); - } - finally //ensure cleanup/shutdown - { - if(DMLScript.USE_ACCELERATOR && ec.getGPUContext() != null) { + rtprog.execute(ec); + } finally { // ensure cleanup/shutdown + if (DMLScript.USE_ACCELERATOR && ec.getGPUContext() != null) { GPUContextPool.returnToPool(ec.getGPUContext()); } - if( dmlconf.getBooleanValue(DMLConfig.CODEGEN) ) + if (dmlconf.getBooleanValue(DMLConfig.CODEGEN)) SpoofCompiler.cleanupCodeGenerator(); - - //display statistics (incl caching stats if enabled) + + // display statistics (incl caching stats if enabled) Statistics.stopRunTimer(); - LOG.info(Statistics.display()); + + MLContext ml = MLContext.getActiveMLContext(); + if ((ml != null) && (ml.isStatistics())) { + int statisticsMaxHeavyHitters = ml.getStatisticsMaxHeavyHitters(); + System.out.println(Statistics.display(statisticsMaxHeavyHitters)); + } } } - + } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a2766394/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java index 7887b7b..272fa0e 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java @@ -694,4 +694,13 @@ public class MLContext { return info().buildTime(); } + /** + * Obtain the maximum number of heavy hitters that are printed out as part + * of the statistics. + * + * @return maximum number of heavy hitters to print + */ + public int getStatisticsMaxHeavyHitters() { + return statisticsMaxHeavyHitters; + } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a2766394/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java index c70fbcf..ed6949f 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java @@ -400,7 +400,7 @@ public class ScriptExecutor { */ protected void executeRuntimeProgram() { try { - ScriptExecutorUtils.executeRuntimeProgram(runtimeProgram, executionContext, config); + ScriptExecutorUtils.executeRuntimeProgram(this); } catch (DMLRuntimeException e) { throw new MLContextException("Exception occurred while executing runtime program", e); } @@ -611,6 +611,11 @@ public class ScriptExecutor { this.statistics = statistics; } + /** + * Set the maximum number of heavy hitters to display with statistics. + * + * @param maxHeavyHitters the maximum number of heavy hitters + */ public void setStatisticsMaxHeavyHitters(int maxHeavyHitters) { this.statisticsMaxHeavyHitters = maxHeavyHitters; } @@ -667,21 +672,31 @@ public class ScriptExecutor { } /** - * Whether or not to enable GPU usage + * Whether or not to enable GPU usage. + * * @param enabled - * true if enabled, false otherwise + * {@code true} if enabled, {@code false} otherwise */ public void setGPU(boolean enabled) { this.gpu = enabled; } /** - * Whether or not to force GPU usage + * Whether or not to force GPU usage. + * * @param enabled - * true if enabled, false otherwise + * {@code true} if enabled, {@code false} otherwise */ public void setForceGPU(boolean enabled) { this.forceGPU = enabled; } + /** + * Obtain the SystemML configuration properties. + * + * @return the configuration properties + */ + public DMLConfig getConfig() { + return config; + } }
