Repository: systemml Updated Branches: refs/heads/master 2d57dc576 -> 94f1b72ef
[SYSTEMML-2132] Fix intermediate memory estimate matrix multiply hops This patch fixes a long hidden issue of unnecessarily large intermediate memory estimate for aggregate binary (i.e., matrix multiply) operators. This estimate is supposed to account for a potential switch to sparse (the worst case at the sparsity turn point) but because we used this turn point, it was already estimated as dense. With this patch, we no longer compile unnecessary spark instructions for auto-encoder over a 100K x 10K dense input with 35GB driver. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/206e912c Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/206e912c Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/206e912c Branch: refs/heads/master Commit: 206e912c5bf36e27fa07e59ada1a9f3a0abe63dd Parents: 2d57dc5 Author: Matthias Boehm <[email protected]> Authored: Mon Feb 5 17:19:50 2018 -0800 Committer: Matthias Boehm <[email protected]> Committed: Mon Feb 5 19:10:46 2018 -0800 ---------------------------------------------------------------------- src/main/java/org/apache/sysml/hops/AggBinaryOp.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/206e912c/src/main/java/org/apache/sysml/hops/AggBinaryOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java index a7b6599..6e87cfb 100644 --- a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java +++ b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java @@ -52,6 +52,7 @@ import org.apache.sysml.runtime.matrix.MatrixCharacteristics; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.mapred.DistributedCacheInput; import org.apache.sysml.runtime.matrix.mapred.MMCJMRReducerWithAggregator; +import org.apache.sysml.runtime.util.UtilFunctions; /* Aggregate binary (cell operations): Sum (aij + bij) @@ -385,8 +386,9 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop //account for potential final dense-sparse transformation (worst-case sparse representation) if( dim2 >= 2 ) //vectors always dense - ret += OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, MatrixBlock.SPARSITY_TURN_POINT); - + ret += MatrixBlock.estimateSizeSparseInMemory(dim1, dim2, + MatrixBlock.SPARSITY_TURN_POINT - UtilFunctions.DOUBLE_EPS); + return ret; }
