Repository: systemml Updated Branches: refs/heads/master 3fd8e495e -> 2663ccd41
[MINOR] fix for SYSTEMML_1795 The GPUContextPool.AVAILABLE_GPUS is read after the lops are constructed, but the value needs to be read before. This patch is a fix that problem. Closes #592 Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/2663ccd4 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/2663ccd4 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/2663ccd4 Branch: refs/heads/master Commit: 2663ccd417e59908c3a461adfd217b667b58ea2d Parents: 3fd8e49 Author: Nakul Jindal <[email protected]> Authored: Wed Jul 26 15:37:06 2017 -0700 Committer: Nakul Jindal <[email protected]> Committed: Wed Jul 26 15:37:06 2017 -0700 ---------------------------------------------------------------------- bin/systemml-standalone.py | 3 ++- src/main/java/org/apache/sysml/api/DMLScript.java | 6 +++++- src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java | 2 -- .../java/org/apache/sysml/api/mlcontext/ScriptExecutor.java | 4 ++++ src/main/java/org/apache/sysml/conf/DMLConfig.java | 5 +++-- .../sysml/runtime/instructions/gpu/context/GPUContextPool.java | 2 +- 6 files changed, 15 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/2663ccd4/bin/systemml-standalone.py ---------------------------------------------------------------------- diff --git a/bin/systemml-standalone.py b/bin/systemml-standalone.py index 367bcdf..a0ee8db 100755 --- a/bin/systemml-standalone.py +++ b/bin/systemml-standalone.py @@ -151,7 +151,8 @@ systemml_default_java_opts = \ '-Xmx8g -Xms4g -Xmn1g ' + \ '-cp ' + classpath + ' ' + \ '-Dlog4j.configuration=file:' + log4j_properties_path + ' ' \ - '-Duser.dir=' + user_dir + '-Duser.dir=' + user_dir +# '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8111' # Reads in key-value pairs from the conf/systemml-env.sh file http://git-wip-us.apache.org/repos/asf/systemml/blob/2663ccd4/src/main/java/org/apache/sysml/api/DMLScript.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java index f428aa2..9cb5ebe 100644 --- a/src/main/java/org/apache/sysml/api/DMLScript.java +++ b/src/main/java/org/apache/sysml/api/DMLScript.java @@ -85,6 +85,7 @@ import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext; import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter; import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer; import org.apache.sysml.runtime.controlprogram.parfor.util.IDHandler; +import org.apache.sysml.runtime.instructions.gpu.context.GPUContextPool; import org.apache.sysml.runtime.io.IOUtilFunctions; import org.apache.sysml.runtime.matrix.CleanupMR; import org.apache.sysml.runtime.matrix.mapred.MRConfigurationNames; @@ -659,13 +660,16 @@ public class DMLScript //print basic time and environment info printStartExecInfo( dmlScriptStr ); - //Step 1: parse configuration files + //Step 1: parse configuration files & write any configuration specific global variables DMLConfig dmlconf = DMLConfig.readConfigurationFile(fnameOptConfig); ConfigurationManager.setGlobalConfig(dmlconf); CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf); ConfigurationManager.setGlobalConfig(cconf); LOG.debug("\nDML config: \n" + dmlconf.getConfigInfo()); + // Sets the GPUs to use for this process (a range, all GPUs, comma separated list or a specific GPU) + GPUContextPool.AVAILABLE_GPUS = dmlconf.getTextValue(DMLConfig.AVAILABLE_GPUS); + //Step 2: set local/remote memory if requested (for compile in AM context) if( dmlconf.getBooleanValue(DMLConfig.YARN_APPMASTER) ){ DMLAppMasterUtils.setupConfigRemoteMaxMemory(dmlconf); http://git-wip-us.apache.org/repos/asf/systemml/blob/2663ccd4/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java b/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java index ebbcc21..389f661 100644 --- a/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java +++ b/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java @@ -79,8 +79,6 @@ public class ScriptExecutorUtils { LibMatrixDNN.DISPLAY_STATISTICS = dmlconf.getBooleanValue(DMLConfig.EXTRA_DNN_STATS); DMLScript.FINEGRAINED_STATISTICS = dmlconf.getBooleanValue(DMLConfig.EXTRA_FINEGRAINED_STATS); - // Sets the GPUs to use for this process (a range, all GPUs, comma separated list or a specific GPU) - GPUContextPool.AVAILABLE_GPUS = dmlconf.getTextValue(DMLConfig.AVAILABLE_GPUS); Statistics.startRunTimer(); try { // run execute (w/ exception handling to ensure proper shutdown) http://git-wip-us.apache.org/repos/asf/systemml/blob/2663ccd4/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java index 1a5d0bb..5665308 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java @@ -51,6 +51,7 @@ import org.apache.sysml.runtime.controlprogram.Program; import org.apache.sysml.runtime.controlprogram.caching.CacheStatistics; import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory; +import org.apache.sysml.runtime.instructions.gpu.context.GPUContextPool; import org.apache.sysml.utils.Explain; import org.apache.sysml.utils.Explain.ExplainCounts; import org.apache.sysml.utils.Explain.ExplainType; @@ -249,6 +250,9 @@ public class ScriptExecutor { oldGPU = DMLScript.USE_ACCELERATOR; DMLScript.USE_ACCELERATOR = gpu; DMLScript.STATISTICS_COUNT = statisticsMaxHeavyHitters; + + // Sets the GPUs to use for this process (a range, all GPUs, comma separated list or a specific GPU) + GPUContextPool.AVAILABLE_GPUS = ConfigurationManager.getDMLConfig().getTextValue(DMLConfig.AVAILABLE_GPUS); } /** http://git-wip-us.apache.org/repos/asf/systemml/blob/2663ccd4/src/main/java/org/apache/sysml/conf/DMLConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/conf/DMLConfig.java b/src/main/java/org/apache/sysml/conf/DMLConfig.java index 415bb57..37db080 100644 --- a/src/main/java/org/apache/sysml/conf/DMLConfig.java +++ b/src/main/java/org/apache/sysml/conf/DMLConfig.java @@ -80,7 +80,7 @@ public class DMLConfig public static final String EXTRA_FINEGRAINED_STATS = "systemml.stats.finegrained"; //boolean public static final String EXTRA_GPU_STATS = "systemml.stats.extraGPU"; //boolean public static final String EXTRA_DNN_STATS = "systemml.stats.extraDNN"; //boolean - public static final String AVAILABLE_GPUS = "systemml.gpu.availableGPUs"; // String to specify which GPUs to use (a range, all GPUs, comma separated list or a specific GPU) + public static final String AVAILABLE_GPUS = "systemml.gpu.availableGPUs"; // String to specify which GPUs to use (a range, all GPUs, comma separated list or a specific GPU) // Fraction of available memory to use. The available memory is computer when the GPUContext is created // to handle the tradeoff on calling cudaMemGetInfo too often. @@ -408,7 +408,8 @@ public class DMLConfig YARN_APPMASTER, YARN_APPMASTERMEM, YARN_MAPREDUCEMEM, CP_PARALLEL_OPS, CP_PARALLEL_IO, NATIVE_BLAS, COMPRESSED_LINALG, CODEGEN, CODEGEN_LITERALS, CODEGEN_PLANCACHE, - EXTRA_GPU_STATS, EXTRA_DNN_STATS, EXTRA_FINEGRAINED_STATS + EXTRA_GPU_STATS, EXTRA_DNN_STATS, EXTRA_FINEGRAINED_STATS, + AVAILABLE_GPUS }; StringBuilder sb = new StringBuilder(); http://git-wip-us.apache.org/repos/asf/systemml/blob/2663ccd4/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUContextPool.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUContextPool.java b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUContextPool.java index 88bf403..a9b1333 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUContextPool.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUContextPool.java @@ -167,7 +167,7 @@ public class GPUContextPool { public static ArrayList<Integer> parseListString(String str, int max) { ArrayList<Integer> result = new ArrayList<>(); str = str.trim(); - if (str == "-1") { // all + if (str.equalsIgnoreCase("-1")) { // all for (int i=0; i<max; i++){ result.add(i); }
