[3/4] incubator-systemml git commit: [SYSTEMML-490] Switch default exec mode to hybrid_spark in spark driver

[mboehm7]

[SYSTEMML-490] Switch default exec mode to hybrid_spark in spark driver

In the past, we've seen several users running with default execution
mode ("hybrid", i.e., cp+mr) instead of "hybrid_spark" even though
SystemML was invoked through spark_submit. This led to very irritating
behavior of spawning MR jobs from the spark driver. This patch now
automatically switches the default exec mode to hybrid_spark if the
process runs in the spark driver.  

Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/2c236072
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/2c236072
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/2c236072

Branch: refs/heads/master
Commit: 2c236072eecba3264df42486e0d762038dea6a49
Parents: 870da0f
Author: Matthias Boehm <mbo...@us.ibm.com>
Authored: Thu Jun 2 23:37:49 2016 -0700
Committer: Matthias Boehm <mbo...@us.ibm.com>
Committed: Thu Jun 2 23:37:49 2016 -0700

----------------------------------------------------------------------
 src/main/java/org/apache/sysml/api/DMLScript.java   |  2 +-
 .../java/org/apache/sysml/hops/OptimizerUtils.java  | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/2c236072/src/main/java/org/apache/sysml/api/DMLScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java 
b/src/main/java/org/apache/sysml/api/DMLScript.java
index 570fb07..d952d78 100644
--- a/src/main/java/org/apache/sysml/api/DMLScript.java
+++ b/src/main/java/org/apache/sysml/api/DMLScript.java
@@ -96,7 +96,7 @@ public class DMLScript
                SPARK                   // execute matrix operations in Spark
        }
        
-       public static RUNTIME_PLATFORM rtplatform = RUNTIME_PLATFORM.HYBRID; 
//default exec mode
+       public static RUNTIME_PLATFORM rtplatform = 
OptimizerUtils.getDefaultExecutionMode();
        public static boolean STATISTICS = false; //default statistics
        public static boolean ENABLE_DEBUG_MODE = false; //default debug mode
        public static boolean USE_LOCAL_SPARK_CONFIG = false; //set default 
local spark configuration - used for local testing

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/2c236072/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java 
b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
index e9beafb..48a4a9b 100644
--- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
@@ -544,6 +544,22 @@ public class OptimizerUtils
         * 
         * @return
         */
+       public static RUNTIME_PLATFORM getDefaultExecutionMode() {
+               //default execution type is hybrid (cp+mr)
+               RUNTIME_PLATFORM ret = RUNTIME_PLATFORM.HYBRID;
+               
+               //switch default to hybrid_spark (cp+spark) if in spark driver
+               String sparkenv = System.getenv().get("SPARK_ENV_LOADED");
+               if( sparkenv != null && sparkenv.equals("1") )
+                       ret = RUNTIME_PLATFORM.HYBRID_SPARK;
+               
+               return ret;
+       }
+       
+       /**
+        * 
+        * @return
+        */
        public static boolean isSparkExecutionMode() {
                return (   DMLScript.rtplatform == RUNTIME_PLATFORM.SPARK
                                || DMLScript.rtplatform == 
RUNTIME_PLATFORM.HYBRID_SPARK);