Repository: systemml Updated Branches: refs/heads/master 8f8743c81 -> 180c4f281
[SYSTEMML-445] Added support for sysml.gpu.memory.util.factor property Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/180c4f28 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/180c4f28 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/180c4f28 Branch: refs/heads/master Commit: 180c4f281153e80fad6340d8bf0d31454d0cb0a0 Parents: 8f8743c Author: Niketan Pansare <[email protected]> Authored: Thu Aug 2 16:20:08 2018 -0700 Committer: Niketan Pansare <[email protected]> Committed: Thu Aug 2 16:20:08 2018 -0700 ---------------------------------------------------------------------- conf/SystemML-config.xml.template | 6 +++++- src/main/java/org/apache/sysml/api/DMLScript.java | 1 + .../org/apache/sysml/api/ScriptExecutorUtils.java | 14 ++++++++++---- .../instructions/gpu/context/GPUMemoryManager.java | 9 +-------- 4 files changed, 17 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/180c4f28/conf/SystemML-config.xml.template ---------------------------------------------------------------------- diff --git a/conf/SystemML-config.xml.template b/conf/SystemML-config.xml.template index 033aadb..ca1c17b 100644 --- a/conf/SystemML-config.xml.template +++ b/conf/SystemML-config.xml.template @@ -96,7 +96,7 @@ <!-- the floating point precision. supported values are double, single --> <sysml.floating.point.precision>double</sysml.floating.point.precision> - <!-- the eviction policy for the GPU bufferpool. supported values are lru, mru, lfu, min_evict, align_memory --> + <!-- the eviction policy for the GPU bufferpool. Supported values are lru, mru, lfu, min_evict, align_memory --> <sysml.gpu.eviction.policy>align_memory</sysml.gpu.eviction.policy> <!-- maximum wrap length for instruction and miscellaneous timer column of statistics --> @@ -108,4 +108,8 @@ <!-- Advanced optimization: fraction of driver memory to use for GPU shadow buffer. This optimization is ignored for double precision. By default, it is disabled (hence set to 0.0). If you intend to train network larger than GPU memory size, consider using single precision and setting this to 0.1 --> <sysml.gpu.eviction.shadow.bufferSize>0.0</sysml.gpu.eviction.shadow.bufferSize> + + <!-- Fraction of available GPU memory to use. This is similar to TensorFlow's per_process_gpu_memory_fraction configuration property. (default: 0.9) --> + <sysml.gpu.memory.util.factor>0.9</sysml.gpu.memory.util.factor> + </root> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/180c4f28/src/main/java/org/apache/sysml/api/DMLScript.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java index bfc9da5..fd0b861 100644 --- a/src/main/java/org/apache/sysml/api/DMLScript.java +++ b/src/main/java/org/apache/sysml/api/DMLScript.java @@ -124,6 +124,7 @@ public class DMLScript public static boolean PRINT_GPU_MEMORY_INFO = false; // whether to print GPU memory-related information public static long EVICTION_SHADOW_BUFFER_MAX_BYTES = 0; // maximum number of bytes to use for shadow buffer public static long EVICTION_SHADOW_BUFFER_CURR_BYTES = 0; // number of bytes to use for shadow buffer + public static double GPU_MEMORY_UTILIZATION_FACTOR = 0.9; // fraction of available GPU memory to use /** * Global variable indicating the script type (DML or PYDML). Can be used http://git-wip-us.apache.org/repos/asf/systemml/blob/180c4f28/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java b/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java index 13d0c78..7a97fcf 100644 --- a/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java +++ b/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java @@ -78,16 +78,22 @@ public class ScriptExecutorUtils { // Whether extra statistics useful for developers and others interested // in digging into performance problems are recorded and displayed DMLScript.FINEGRAINED_STATISTICS = DMLScript.STATISTICS && dmlconf.getBooleanValue(DMLConfig.EXTRA_FINEGRAINED_STATS); - DMLScript.PRINT_GPU_MEMORY_INFO = dmlconf.getBooleanValue(DMLConfig.PRINT_GPU_MEMORY_INFO); - DMLScript.SYNCHRONIZE_GPU = dmlconf.getBooleanValue(DMLConfig.SYNCHRONIZE_GPU); CacheableData.CACHING_BUFFER_SIZE = dmlconf.getDoubleValue(DMLConfig.CACHING_BUFFER_SIZE); if(CacheableData.CACHING_BUFFER_SIZE < 0 || CacheableData.CACHING_BUFFER_SIZE > 1) throw new RuntimeException("Incorrect value (" + CacheableData.CACHING_BUFFER_SIZE + ") for the configuration " + DMLConfig.CACHING_BUFFER_SIZE); - DMLScript.EAGER_CUDA_FREE = dmlconf.getBooleanValue(DMLConfig.EAGER_CUDA_FREE); + DMLScript.STATISTICS_MAX_WRAP_LEN = dmlconf.getIntValue(DMLConfig.STATS_MAX_WRAP_LEN); NativeHelper.initialize(dmlconf.getTextValue(DMLConfig.NATIVE_BLAS_DIR), dmlconf.getTextValue(DMLConfig.NATIVE_BLAS).trim()); if(DMLScript.USE_ACCELERATOR) { + DMLScript.SYNCHRONIZE_GPU = dmlconf.getBooleanValue(DMLConfig.SYNCHRONIZE_GPU); + DMLScript.EAGER_CUDA_FREE = dmlconf.getBooleanValue(DMLConfig.EAGER_CUDA_FREE); + DMLScript.PRINT_GPU_MEMORY_INFO = dmlconf.getBooleanValue(DMLConfig.PRINT_GPU_MEMORY_INFO); + DMLScript.GPU_MEMORY_UTILIZATION_FACTOR = dmlconf.getDoubleValue(DMLConfig.GPU_MEMORY_UTILIZATION_FACTOR); + if(DMLScript.GPU_MEMORY_UTILIZATION_FACTOR < 0 || DMLScript.GPU_MEMORY_UTILIZATION_FACTOR > 1) { + throw new RuntimeException("Incorrect value (" + DMLScript.GPU_MEMORY_UTILIZATION_FACTOR + ") for the configuration:" + DMLConfig.GPU_MEMORY_UTILIZATION_FACTOR); + } + DMLScript.FLOATING_POINT_PRECISION = dmlconf.getTextValue(DMLConfig.FLOATING_POINT_PRECISION); org.apache.sysml.runtime.matrix.data.LibMatrixCUDA.resetFloatingPointPrecision(); if(DMLScript.FLOATING_POINT_PRECISION.equals("double")) { @@ -96,7 +102,7 @@ public class ScriptExecutorUtils { else { double shadowBufferSize = dmlconf.getDoubleValue(DMLConfig.EVICTION_SHADOW_BUFFERSIZE); if(shadowBufferSize < 0 || shadowBufferSize > 1) - throw new RuntimeException("Incorrect value (" + shadowBufferSize + ") for the configuration " + DMLConfig.EVICTION_SHADOW_BUFFERSIZE); + throw new RuntimeException("Incorrect value (" + shadowBufferSize + ") for the configuration:" + DMLConfig.EVICTION_SHADOW_BUFFERSIZE); DMLScript.EVICTION_SHADOW_BUFFER_MAX_BYTES = (long) (((double)InfrastructureAnalyzer.getLocalMaxMemory())*shadowBufferSize); if(DMLScript.EVICTION_SHADOW_BUFFER_MAX_BYTES > 0 && DMLScript.EVICTION_SHADOW_BUFFER_CURR_BYTES > DMLScript.EVICTION_SHADOW_BUFFER_MAX_BYTES) { http://git-wip-us.apache.org/repos/asf/systemml/blob/180c4f28/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUMemoryManager.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUMemoryManager.java b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUMemoryManager.java index acfba66..c4ae253 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUMemoryManager.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUMemoryManager.java @@ -37,8 +37,6 @@ import java.util.stream.Collectors; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.sysml.api.DMLScript; -import org.apache.sysml.conf.ConfigurationManager; -import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.instructions.gpu.GPUInstruction; @@ -130,11 +128,6 @@ public class GPUMemoryManager { // This often happens if user tries to use both TF and SystemML, and TF grabs onto 90% of the memory ahead of time. private static final double WARN_UTILIZATION_FACTOR = 0.7; - // Invoke cudaMemGetInfo to get available memory information. Useful if GPU is shared among multiple application. - public double GPU_MEMORY_UTILIZATION_FACTOR = ConfigurationManager.getDMLConfig() - .getDoubleValue(DMLConfig.GPU_MEMORY_UTILIZATION_FACTOR); - - public GPUMemoryManager(GPUContext gpuCtx) { matrixMemoryManager = new GPUMatrixMemoryManager(this); lazyCudaFreeMemoryManager = new GPULazyCudaFreeMemoryManager(this); @@ -603,7 +596,7 @@ public class GPUMemoryManager { long free[] = { 0 }; long total[] = { 0 }; cudaMemGetInfo(free, total); - return (long) (free[0] * GPU_MEMORY_UTILIZATION_FACTOR); + return (long) (free[0] * DMLScript.GPU_MEMORY_UTILIZATION_FACTOR); } private static class CustomPointer extends Pointer {
