[SYSTEMML-930] Warn for result size when call through MLContext
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/c8076d04 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/c8076d04 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/c8076d04 Branch: refs/heads/master Commit: c8076d04a2b2512614a91c22b05c8c5599922f5f Parents: 4bc6601 Author: Arvind Surve <ac...@yahoo.com> Authored: Wed Sep 21 22:22:38 2016 -0700 Committer: Arvind Surve <ac...@yahoo.com> Committed: Wed Sep 21 22:26:59 2016 -0700 ---------------------------------------------------------------------- .../context/SparkExecutionContext.java | 10 +++++++++- .../apache/sysml/runtime/util/UtilFunctions.java | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8076d04/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java index f95e0d4..a6f99e8 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java @@ -233,7 +233,15 @@ public class SparkExecutionContext extends ExecutionContext _spctx = new JavaSparkContext(conf); } } - + + // Set warning if spark.driver.maxResultSize is not set. It needs to be set before starting Spark Context for CP collect + String strDriverMaxResSize = _spctx.getConf().get("spark.driver.maxResultSize", "1g"); + long driverMaxResSize = UtilFunctions.parseMemorySize(strDriverMaxResSize); + if (driverMaxResSize != 0 && driverMaxResSize<OptimizerUtils.getLocalMemBudget()) + LOG.warn("Configuration parameter spark.driver.maxResultSize set to " + UtilFunctions.formatMemorySize(driverMaxResSize) + "." + + " You can set it through Spark default configuration setting either to 0 (unlimited) or to available memory budget of size " + + UtilFunctions.formatMemorySize((long)OptimizerUtils.getLocalMemBudget()) + "."); + //globally add binaryblock serialization framework for all hdfs read/write operations //TODO if spark context passed in from outside (mlcontext), we need to clean this up at the end if( MRJobConfiguration.USE_BINARYBLOCK_SERIALIZATION ) http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8076d04/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java index 81a1e8f..e5a792a 100644 --- a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java +++ b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java @@ -556,6 +556,24 @@ public class UtilFunctions } /** + * Format a memory size with g/m/k quantifiers into its + * number representation. + * + * @param arg + * @return + */ + public static String formatMemorySize(long arg) { + if (arg >= 1024 * 1024 * 1024) + return String.format("%d GB", arg/(1024*1024*1024)); + else if (arg >= 1024 * 1024) + return String.format("%d MB", arg/(1024*1024)); + else if (arg >= 1024) + return String.format("%d KB", arg/(1024)); + else + return String.format("%d", arg); + } + + /** * * @param low lower bound (inclusive) * @param up upper bound (inclusive)