[SYSTEMML-2048] Compiler integration large dense blocks (no fallbacks) After modifying all runtime operations to work with large dense blocks, this patch now integrate these blocks into the compiler. So far, we compiled distributed operations if an intermediate was potentially dense and larger than 16GB in dense representation. With the new block runtime, this fallback to distributed operations is unnecessary.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/20b1b5a9 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/20b1b5a9 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/20b1b5a9 Branch: refs/heads/master Commit: 20b1b5a9cfc9506c73a2d10b776048485c1db5e8 Parents: 0a9c91a Author: Matthias Boehm <[email protected]> Authored: Wed Jan 10 18:42:00 2018 -0800 Committer: Matthias Boehm <[email protected]> Committed: Wed Jan 10 18:42:15 2018 -0800 ---------------------------------------------------------------------- src/main/java/org/apache/sysml/hops/Hop.java | 24 ++++------------------- 1 file changed, 4 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/20b1b5a9/src/main/java/org/apache/sysml/hops/Hop.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java index 0a3972b..667cd09 100644 --- a/src/main/java/org/apache/sysml/hops/Hop.java +++ b/src/main/java/org/apache/sysml/hops/Hop.java @@ -97,7 +97,6 @@ public abstract class Hop implements ParseInfo protected double _memEstimate = OptimizerUtils.INVALID_SIZE; protected double _processingMemEstimate = 0; protected double _spBroadcastMemEstimate = 0; - protected boolean _validCPSizeEstimate = false; // indicates if there are unknowns during compilation // (in that case re-complication ensures robustness and efficiency) @@ -215,24 +214,13 @@ public abstract class Hop implements ParseInfo } public void checkAndSetInvalidCPDimsAndSize() - { - if( _etype == ExecType.CP ) - { - boolean invalid = false; - - //Step 1: check dimensions of output and all inputs (INTEGER) - invalid |= !OptimizerUtils.isValidCPDimensions(_dim1, _dim2); + { + if( _etype == ExecType.CP || _etype == ExecType.GPU ) { + //check dimensions of output and all inputs (INTEGER) + boolean invalid = !OptimizerUtils.isValidCPDimensions(_dim1, _dim2); for( Hop in : getInput() ) invalid |= !OptimizerUtils.isValidCPDimensions(in._dim1, in._dim2); - //Step 2: check valid output and input sizes for cp (<16GB for DENSE) - //(if the memory estimate is smaller than max_numcells we are guaranteed to have it in sparse representation) - invalid |= !( OptimizerUtils.isValidCPMatrixSize(_dim1, _dim2, OptimizerUtils.getSparsity(_dim1, _dim2, _nnz)) - || getOutputMemEstimate() < 8*OptimizerUtils.MAX_NUMCELLS_CP_DENSE || _validCPSizeEstimate ); - for( Hop in : getInput() ) - invalid |= !( OptimizerUtils.isValidCPMatrixSize(in._dim1, in._dim2, OptimizerUtils.getSparsity(in._dim1, in._dim2, in._nnz)) - || in.getOutputMemEstimate() < 8*OptimizerUtils.MAX_NUMCELLS_CP_DENSE || in._validCPSizeEstimate); - //force exec type mr if necessary if( invalid ) { if( DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID ) @@ -712,10 +700,6 @@ public abstract class Hop implements ParseInfo //final estimate (sum of inputs/intermediates/output) _memEstimate = getInputOutputSize(); - - //update optional valid cp size estimate (based on worst-case dimensions) - _validCPSizeEstimate = (wstats!=null) ? OptimizerUtils.isValidCPMatrixSize( - wstats[0], wstats[1], OptimizerUtils.getSparsity(wstats[0], wstats[1], wstats[2])) : false; } /**
