Repository: systemml
Updated Branches:
  refs/heads/master 2604bb3f1 -> b0910631e


[SYSTEMML-2170] Fix parfor optimizer exec type selection (valid dims)

This patch fixes the parfor optimizer rewrite for selecting the
execution mode (local vs remote parfor) to properly account for valid CP
dimensions in addition to the satisfied memory budget constraints.
Previously, we did not check the dimensions, which led to invalid
decisions and thus crashes for ultra-sparse matrices with dimensions
larger than max integer but small memory requirements.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/b0910631
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/b0910631
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/b0910631

Branch: refs/heads/master
Commit: b0910631e1a77267ab436ba5d2da8be412874a82
Parents: 2604bb3
Author: Matthias Boehm <mboe...@gmail.com>
Authored: Tue Mar 6 23:48:54 2018 -0800
Committer: Matthias Boehm <mboe...@gmail.com>
Committed: Tue Mar 6 23:48:54 2018 -0800

----------------------------------------------------------------------
 src/main/java/org/apache/sysml/hops/Hop.java             | 11 ++++++++---
 .../controlprogram/parfor/opt/OptimizerRuleBased.java    |  3 ++-
 2 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/systemml/blob/b0910631/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java 
b/src/main/java/org/apache/sysml/hops/Hop.java
index 71f4d89..5871047 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -219,9 +219,7 @@ public abstract class Hop implements ParseInfo
        {
                if( _etype == ExecType.CP || _etype == ExecType.GPU ) {
                        //check dimensions of output and all inputs (INTEGER)
-                       boolean invalid = 
!OptimizerUtils.isValidCPDimensions(_dim1, _dim2);
-                       for( Hop in : getInput() )
-                               invalid |= 
!OptimizerUtils.isValidCPDimensions(in._dim1, in._dim2);
+                       boolean invalid = !hasValidCPDimsAndSize();
                        
                        //force exec type mr if necessary
                        if( invalid ) { 
@@ -232,6 +230,13 @@ public abstract class Hop implements ParseInfo
                        }
                }
        }
+       
+       public boolean hasValidCPDimsAndSize() {
+               boolean invalid = !OptimizerUtils.isValidCPDimensions(_dim1, 
_dim2);
+               for( Hop in : getInput() )
+                       invalid |= 
!OptimizerUtils.isValidCPDimensions(in._dim1, in._dim2);
+               return !invalid;
+       }
 
        public boolean hasMatrixInputWithDifferentBlocksizes()
        {

http://git-wip-us.apache.org/repos/asf/systemml/blob/b0910631/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git 
a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
 
b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index 0359f7f..49e619d 100644
--- 
a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ 
b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -907,7 +907,8 @@ public class OptimizerRuleBased extends Optimizer
                {
                        Hop h = 
OptTreeConverter.getAbstractPlanMapping().getMappedHop( n.getID() );
                        if(    h.getForcedExecType()!=LopProperties.ExecType.MR 
 //e.g., -exec=hadoop
-                               && 
h.getForcedExecType()!=LopProperties.ExecType.SPARK) 
+                               && 
h.getForcedExecType()!=LopProperties.ExecType.SPARK 
+                               && h.hasValidCPDimsAndSize() ) //integer dims
                        {
                                double mem = 
_cost.getLeafNodeEstimate(TestMeasure.MEMORY_USAGE, n, 
LopProperties.ExecType.CP);
                                if( mem <= memBudget )

Reply via email to