Repository: systemml Updated Branches: refs/heads/master 2604bb3f1 -> b0910631e
[SYSTEMML-2170] Fix parfor optimizer exec type selection (valid dims) This patch fixes the parfor optimizer rewrite for selecting the execution mode (local vs remote parfor) to properly account for valid CP dimensions in addition to the satisfied memory budget constraints. Previously, we did not check the dimensions, which led to invalid decisions and thus crashes for ultra-sparse matrices with dimensions larger than max integer but small memory requirements. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/b0910631 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/b0910631 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/b0910631 Branch: refs/heads/master Commit: b0910631e1a77267ab436ba5d2da8be412874a82 Parents: 2604bb3 Author: Matthias Boehm <[email protected]> Authored: Tue Mar 6 23:48:54 2018 -0800 Committer: Matthias Boehm <[email protected]> Committed: Tue Mar 6 23:48:54 2018 -0800 ---------------------------------------------------------------------- src/main/java/org/apache/sysml/hops/Hop.java | 11 ++++++++--- .../controlprogram/parfor/opt/OptimizerRuleBased.java | 3 ++- 2 files changed, 10 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/b0910631/src/main/java/org/apache/sysml/hops/Hop.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java index 71f4d89..5871047 100644 --- a/src/main/java/org/apache/sysml/hops/Hop.java +++ b/src/main/java/org/apache/sysml/hops/Hop.java @@ -219,9 +219,7 @@ public abstract class Hop implements ParseInfo { if( _etype == ExecType.CP || _etype == ExecType.GPU ) { //check dimensions of output and all inputs (INTEGER) - boolean invalid = !OptimizerUtils.isValidCPDimensions(_dim1, _dim2); - for( Hop in : getInput() ) - invalid |= !OptimizerUtils.isValidCPDimensions(in._dim1, in._dim2); + boolean invalid = !hasValidCPDimsAndSize(); //force exec type mr if necessary if( invalid ) { @@ -232,6 +230,13 @@ public abstract class Hop implements ParseInfo } } } + + public boolean hasValidCPDimsAndSize() { + boolean invalid = !OptimizerUtils.isValidCPDimensions(_dim1, _dim2); + for( Hop in : getInput() ) + invalid |= !OptimizerUtils.isValidCPDimensions(in._dim1, in._dim2); + return !invalid; + } public boolean hasMatrixInputWithDifferentBlocksizes() { http://git-wip-us.apache.org/repos/asf/systemml/blob/b0910631/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java index 0359f7f..49e619d 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java @@ -907,7 +907,8 @@ public class OptimizerRuleBased extends Optimizer { Hop h = OptTreeConverter.getAbstractPlanMapping().getMappedHop( n.getID() ); if( h.getForcedExecType()!=LopProperties.ExecType.MR //e.g., -exec=hadoop - && h.getForcedExecType()!=LopProperties.ExecType.SPARK) + && h.getForcedExecType()!=LopProperties.ExecType.SPARK + && h.hasValidCPDimsAndSize() ) //integer dims { double mem = _cost.getLeafNodeEstimate(TestMeasure.MEMORY_USAGE, n, LopProperties.ExecType.CP); if( mem <= memBudget )
