[SYSTEMML-1427] Fix issues of worst-case and parfor memory estimates This patch fixes various special cases of worst-case and parfor memory estimates, including (1) incomplete worst-case estimates, (2) parfor w/ spark execution type, (3) constrained parfor optimizer with unspecified degree of parallelism and/or execution type.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/f693bcac Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/f693bcac Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/f693bcac Branch: refs/heads/master Commit: f693bcac7d6968452fc2e07c9a06e6b7fe1cbcdd Parents: 128acb3 Author: Matthias Boehm <[email protected]> Authored: Mon Mar 20 15:21:57 2017 -0700 Committer: Matthias Boehm <[email protected]> Committed: Mon Mar 20 15:21:57 2017 -0700 ---------------------------------------------------------------------- src/main/java/org/apache/sysml/hops/Hop.java | 2 +- .../runtime/controlprogram/parfor/opt/CostEstimator.java | 10 ++++++---- .../controlprogram/parfor/opt/CostEstimatorHops.java | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f693bcac/src/main/java/org/apache/sysml/hops/Hop.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java index 86741ba..cdee2c7 100644 --- a/src/main/java/org/apache/sysml/hops/Hop.java +++ b/src/main/java/org/apache/sysml/hops/Hop.java @@ -618,7 +618,7 @@ public abstract class Hop //infer the output stats wstats = inferOutputCharacteristics(memo); - if( wstats != null ) { + if( wstats != null && wstats[0] >= 0 && wstats[1] >= 0 ) { //use worst case characteristics to estimate mem long lnnz = ((wstats[2]>=0)?wstats[2]:wstats[0]*wstats[1]); _outputMemEstimate = computeOutputMemEstimate( wstats[0], wstats[1], lnnz ); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f693bcac/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java index 3fdf8bd..4af7920 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java @@ -169,7 +169,8 @@ public abstract class CostEstimator case PARFOR: tmp = node.getParam(ParamType.NUM_ITERATIONS); N = (tmp!=null) ? (double)Long.parseLong(tmp) : FACTOR_NUM_ITERATIONS; - val = N * getSumEstimate(measure, node.getChilds(), et) / node.getK(); + val = N * getSumEstimate(measure, node.getChilds(), et) + / Math.max(node.getK(), 1); break; default: //do nothing @@ -187,10 +188,11 @@ public abstract class CostEstimator val = getMaxEstimate(measure, node.getChilds(), et); break; case PARFOR: - if( node.getExecType() == OptNode.ExecType.MR ) + if( node.getExecType() == OptNode.ExecType.MR || node.getExecType() == OptNode.ExecType.SPARK ) val = getMaxEstimate(measure, node.getChilds(), et); //executed in different JVMs - else if ( node.getExecType() == OptNode.ExecType.CP ) - val = getMaxEstimate(measure, node.getChilds(), et) * node.getK(); //everything executed within 1 JVM + else if ( node.getExecType() == OptNode.ExecType.CP || node.getExecType() == null ) + val = getMaxEstimate(measure, node.getChilds(), et) + * Math.max(node.getK(), 1); //everything executed within 1 JVM break; default: //do nothing http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f693bcac/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorHops.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorHops.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorHops.java index b65d80f..9646b09 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorHops.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorHops.java @@ -93,7 +93,7 @@ public class CostEstimatorHops extends CostEstimator } //check for forced runtime platform - if( h.getForcedExecType()==ExecType.MR || h.getExecType()==ExecType.SPARK) + if( h.getForcedExecType()==ExecType.MR || h.getForcedExecType()==ExecType.SPARK) { value = DEFAULT_MEM_REMOTE; }
