Repository: systemml Updated Branches: refs/heads/master 62b5725d8 -> 2f87565d8
[MINOR] Updated setBLASPath to provide the BLAS type along with the path Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/2f87565d Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/2f87565d Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/2f87565d Branch: refs/heads/master Commit: 2f87565d8ce5ad385870b8af12f3c3fda53470f2 Parents: 62b5725 Author: Niketan Pansare <[email protected]> Authored: Wed Nov 15 10:04:34 2017 -0800 Committer: Niketan Pansare <[email protected]> Committed: Wed Nov 15 10:04:34 2017 -0800 ---------------------------------------------------------------------- .../org/apache/sysml/utils/NativeHelper.java | 53 ++++++++++---------- src/main/python/systemml/mlcontext.py | 9 ++-- 2 files changed, 32 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/2f87565d/src/main/java/org/apache/sysml/utils/NativeHelper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/NativeHelper.java b/src/main/java/org/apache/sysml/utils/NativeHelper.java index b57cfd6..f70ea14 100644 --- a/src/main/java/org/apache/sysml/utils/NativeHelper.java +++ b/src/main/java/org/apache/sysml/utils/NativeHelper.java @@ -48,7 +48,6 @@ public class NativeHelper { public static String blasType; private static int maxNumThreads = -1; private static boolean setMaxNumThreads = false; - private static String customLibPath = null; static { // Note: we only support 64 bit Java on x86 and AMD machine supportedArchitectures.put("x86_64", "x86_64"); @@ -59,31 +58,24 @@ public class NativeHelper { private static String hintOnFailures = ""; - public static void setBLASPath(String path) { - customLibPath = path; - init(true); + public static void setBLASPath(String customLibPath, String userSpecifiedBLAS) { + init(customLibPath, userSpecifiedBLAS); } // Performing loading in a method instead of a static block will throw a detailed stack trace in case of fatal errors - private static void init(boolean forcedInit) { + private static void init(String customLibPath, String userSpecifiedBLAS) { // Only Linux supported for BLAS if(!SystemUtils.IS_OS_LINUX) return; // attemptedLoading variable ensures that we don't try to load SystemML and other dependencies // again and again especially in the parfor (hence the double-checking with synchronized). - if(!attemptedLoading || forcedInit) { - DMLConfig dmlConfig = ConfigurationManager.getDMLConfig(); + if(!attemptedLoading || customLibPath != null) { // ------------------------------------------------------------------------------------- - // We allow BLAS to be enabled or disabled or explicitly selected in one of the two ways: - // 1. DML Configuration: native.blas (boolean flag) - // 2. Environment variable: SYSTEMML_BLAS (can be set to mkl, openblas or none) - // The option 1 will be removed in later SystemML versions. - // The option 2 is useful for two reasons: - // - Developer testing of different BLAS - // - Provides fine-grained control. Certain machines could use mkl while others use openblas, etc. - String userSpecifiedBLAS = (dmlConfig == null) ? "auto" : dmlConfig.getTextValue(DMLConfig.NATIVE_BLAS).trim().toLowerCase(); - + if(userSpecifiedBLAS == null) { + DMLConfig dmlConfig = ConfigurationManager.getDMLConfig(); + userSpecifiedBLAS = (dmlConfig == null) ? "auto" : dmlConfig.getTextValue(DMLConfig.NATIVE_BLAS).trim().toLowerCase(); + } if(userSpecifiedBLAS.equals("auto") || userSpecifiedBLAS.equals("mkl") || userSpecifiedBLAS.equals("openblas")) { long start = System.nanoTime(); if(!supportedArchitectures.containsKey(SystemUtils.OS_ARCH)) { @@ -91,24 +83,24 @@ public class NativeHelper { return; } synchronized(NativeHelper.class) { - if(!attemptedLoading || forcedInit) { + if(!attemptedLoading || customLibPath != null) { // ----------------------------------------------------------------------------- // ============================================================================= // By default, we will native.blas=true and we will attempt to load MKL first. // If MKL is not enabled then we try to load OpenBLAS. // If both MKL and OpenBLAS are not available we fall back to Java BLAS. if(userSpecifiedBLAS.equals("auto")) { - blasType = isMKLAvailable() ? "mkl" : isOpenBLASAvailable() ? "openblas" : null; + blasType = isMKLAvailable(customLibPath) ? "mkl" : isOpenBLASAvailable(customLibPath) ? "openblas" : null; if(blasType == null) LOG.info("Unable to load either MKL or OpenBLAS due to " + hintOnFailures); } else if(userSpecifiedBLAS.equals("mkl")) { - blasType = isMKLAvailable() ? "mkl" : null; + blasType = isMKLAvailable(customLibPath) ? "mkl" : null; if(blasType == null) LOG.info("Unable to load MKL due to " + hintOnFailures); } else if(userSpecifiedBLAS.equals("openblas")) { - blasType = isOpenBLASAvailable() ? "openblas" : null; + blasType = isOpenBLASAvailable(customLibPath) ? "openblas" : null; if(blasType == null) LOG.info("Unable to load OpenBLAS due to " + hintOnFailures); } @@ -158,7 +150,14 @@ public class NativeHelper { } public static boolean isNativeLibraryLoaded() { - init(false); + // We allow BLAS to be enabled or disabled or explicitly selected in one of the two ways: + // 1. DML Configuration: native.blas (boolean flag) + // 2. Environment variable: SYSTEMML_BLAS (can be set to mkl, openblas or none) + // The option 1 will be removed in later SystemML versions. + // The option 2 is useful for two reasons: + // - Developer testing of different BLAS + // - Provides fine-grained control. Certain machines could use mkl while others use openblas, etc. + init(null, null); if(maxNumThreads == -1) maxNumThreads = OptimizerUtils.getConstrainedNumThreads(-1); if(isSystemMLLoaded && !setMaxNumThreads && maxNumThreads != -1) { @@ -178,17 +177,17 @@ public class NativeHelper { } - private static boolean isMKLAvailable() { - return loadBLAS("mkl_rt", null); + private static boolean isMKLAvailable(String customLibPath) { + return loadBLAS(customLibPath, "mkl_rt", null); } - private static boolean isOpenBLASAvailable() { - if(!loadBLAS("gomp", "gomp required for loading OpenBLAS-enabled SystemML library")) + private static boolean isOpenBLASAvailable(String customLibPath) { + if(!loadBLAS(customLibPath, "gomp", "gomp required for loading OpenBLAS-enabled SystemML library")) return false; - return loadBLAS("openblas", null); + return loadBLAS(customLibPath, "openblas", null); } - private static boolean loadBLAS(String blas, String optionalMsg) { + private static boolean loadBLAS(String customLibPath, String blas, String optionalMsg) { // First attempt to load from custom library path if(customLibPath != null) { String libPath = customLibPath + File.separator + System.mapLibraryName(blas); http://git-wip-us.apache.org/repos/asf/systemml/blob/2f87565d/src/main/python/systemml/mlcontext.py ---------------------------------------------------------------------- diff --git a/src/main/python/systemml/mlcontext.py b/src/main/python/systemml/mlcontext.py index af1a57e..704ba98 100644 --- a/src/main/python/systemml/mlcontext.py +++ b/src/main/python/systemml/mlcontext.py @@ -66,7 +66,7 @@ def _get_spark_context(): -def setBLASPath(path): +def setBLASPath(path, blas='auto'): """ This method useful in the cloud environment where the user doesnot have sudo permission or where setting environment variables @@ -75,10 +75,13 @@ def setBLASPath(path): Parameters ---------- path: String - Custom path where the BLAS libraries where located. + Custom path to the directory where the BLAS shared libraries are located. + + blas: String + Can be auto, openblas or mkl """ sc = _get_spark_context() - sc._jvm.org.apache.sysml.utils.NativeHelper.setBLASPath(path) + sc._jvm.org.apache.sysml.utils.NativeHelper.setBLASPath(str(path), blas)
