Repository: systemml Updated Branches: refs/heads/master 2625cd903 -> 62b64b32d
[SYSTEMML-1596] Set runtime platform via MLContext Update MLContext and ScriptExecutor to allow for convenient setting of the runtime platform. This is useful when debugging and developing. Closes #497. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/62b64b32 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/62b64b32 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/62b64b32 Branch: refs/heads/master Commit: 62b64b32d200817fa320a91b5d5a35751a747856 Parents: 2625cd9 Author: Deron Eriksson <[email protected]> Authored: Mon Jul 17 15:31:29 2017 -0700 Committer: Deron Eriksson <[email protected]> Committed: Mon Jul 17 15:31:29 2017 -0700 ---------------------------------------------------------------------- .../apache/sysml/api/mlcontext/MLContext.java | 63 +++++++++++++++++++- .../sysml/api/mlcontext/ScriptExecutor.java | 24 +++++++- 2 files changed, 83 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/62b64b32/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java index a9ab393..b35faa6 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java @@ -28,7 +28,6 @@ import org.apache.spark.SparkContext; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SparkSession; import org.apache.sysml.api.DMLScript; -import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; import org.apache.sysml.api.jmlc.JMLCUtils; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.conf.DMLConfig; @@ -114,6 +113,12 @@ public class MLContext { private ExplainLevel explainLevel = null; /** + * The runtime platform on which to execute. By default, MLContext runs on + * {@code ExecutionType.DRIVER_AND_SPARK}. + */ + private ExecutionType executionType = ExecutionType.DRIVER_AND_SPARK; + + /** * Whether or not all values should be maintained in the symbol table after * execution. */ @@ -160,6 +165,38 @@ public class MLContext { }; /** + * The different types of execution environments supported by SystemML. The + * default execution type is {@code DRIVER_AND_SPARK}. {@code DRIVER} refers + * to all operations occurring in the local driver JVM. {@code SPARK} refers + * to all operations occurring on Spark. {@code HADOOP} refers to all + * operations occurring on Hadoop. {@code DRIVER_AND_SPARK} refers to + * operations occurring in the local driver JVM and on Spark when + * appropriate. {@code DRIVER_AND_HADOOP} refers to operations occurring in + * the local driver JVM and on Hadoop when appropriate. + * + */ + public enum ExecutionType { + DRIVER, SPARK, HADOOP, DRIVER_AND_SPARK, DRIVER_AND_HADOOP; + + public DMLScript.RUNTIME_PLATFORM getRuntimePlatform() { + switch (this) { + case DRIVER: + return DMLScript.RUNTIME_PLATFORM.SINGLE_NODE; + case SPARK: + return DMLScript.RUNTIME_PLATFORM.SPARK; + case HADOOP: + return DMLScript.RUNTIME_PLATFORM.HADOOP; + case DRIVER_AND_SPARK: + return DMLScript.RUNTIME_PLATFORM.HYBRID_SPARK; + case DRIVER_AND_HADOOP: + return DMLScript.RUNTIME_PLATFORM.HYBRID; + default: + return DMLScript.RUNTIME_PLATFORM.HYBRID_SPARK; + } + } + } + + /** * Retrieve the currently active MLContext. This is used internally by * SystemML via MLContextProxy. * @@ -235,8 +272,7 @@ public class MLContext { } this.spark = spark; - // by default, run in hybrid Spark mode for optimal performance - DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; + DMLScript.rtplatform = executionType.getRuntimePlatform(); activeMLContext = this; MLContextProxy.setActive(true); @@ -279,6 +315,7 @@ public class MLContext { */ public MLResults execute(Script script) { ScriptExecutor scriptExecutor = new ScriptExecutor(); + scriptExecutor.setExecutionType(executionType); scriptExecutor.setExplain(explain); scriptExecutor.setExplainLevel(explainLevel); scriptExecutor.setGPU(gpu); @@ -697,4 +734,24 @@ public class MLContext { this.initBeforeExecution = initBeforeExecution; } + /** + * Obtain the current execution environment. + * + * @return the execution environment + */ + public ExecutionType getExecutionType() { + return executionType; + } + + /** + * Set the execution environment. + * + * @param executionType + * the execution environment + */ + public void setExecutionType(ExecutionType executionType) { + DMLScript.rtplatform = executionType.getRuntimePlatform(); + this.executionType = executionType; + } + } http://git-wip-us.apache.org/repos/asf/systemml/blob/62b64b32/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java index 1a6a8ed..6d19166 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java @@ -25,9 +25,10 @@ import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.apache.sysml.api.DMLScript; -import org.apache.sysml.api.ScriptExecutorUtils; import org.apache.sysml.api.DMLScript.DMLOptions; +import org.apache.sysml.api.ScriptExecutorUtils; import org.apache.sysml.api.jmlc.JMLCUtils; +import org.apache.sysml.api.mlcontext.MLContext.ExecutionType; import org.apache.sysml.api.mlcontext.MLContext.ExplainLevel; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.conf.DMLConfig; @@ -125,6 +126,7 @@ public class ScriptExecutor { protected boolean statistics = false; protected boolean oldStatistics = false; protected ExplainLevel explainLevel; + protected ExecutionType executionType; protected int statisticsMaxHeavyHitters = 10; protected boolean maintainSymbolTable = false; @@ -693,4 +695,24 @@ public class ScriptExecutor { public DMLConfig getConfig() { return config; } + + /** + * Obtain the current execution environment. + * + * @return the execution environment + */ + public ExecutionType getExecutionType() { + return executionType; + } + + /** + * Set the execution environment. + * + * @param executionType + * the execution environment + */ + public void setExecutionType(ExecutionType executionType) { + DMLScript.rtplatform = executionType.getRuntimePlatform(); + this.executionType = executionType; + } }
