Repository: incubator-systemml Updated Branches: refs/heads/master aa6d38c94 -> 4e445d17e
[SYSTEMML-1574] Fix serialized caching of ultra-sparse matrices Previously, we stored all sparse matrices that are expected to exceed aggregate memory in serialized form in order to eliminate unnecessary in-meomry storage overheads. With the introduction of csr we disabled this code path as csr's in-memory representation is as large as its serialized form. However, this is not true for ultra-sparse matrices which still benefit from serialized storage. Accordingly this patch tweaks the corresponding selecting of checkpoint storage levels. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/4e445d17 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/4e445d17 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/4e445d17 Branch: refs/heads/master Commit: 4e445d17e7f06e0c0b04bc2610806fbf17dc68ca Parents: aa6d38c Author: Matthias Boehm <[email protected]> Authored: Wed May 3 11:45:38 2017 -0400 Committer: Matthias Boehm <[email protected]> Committed: Wed May 3 11:45:38 2017 -0400 ---------------------------------------------------------------------- src/main/java/org/apache/sysml/hops/Hop.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4e445d17/src/main/java/org/apache/sysml/hops/Hop.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java index eb9aa5e..6c7089b 100644 --- a/src/main/java/org/apache/sysml/hops/Hop.java +++ b/src/main/java/org/apache/sysml/hops/Hop.java @@ -313,7 +313,6 @@ public abstract class Hop } } - @SuppressWarnings("unused") //see CHECKPOINT_SPARSE_CSR private void constructAndSetCheckpointLopIfRequired() throws HopsException { @@ -345,11 +344,13 @@ public abstract class Hop //investigate need for serialized storage of large sparse matrices //(compile- instead of runtime-level for better debugging) boolean serializedStorage = false; - if( getDataType()==DataType.MATRIX && dimsKnown(true) && !Checkpoint.CHECKPOINT_SPARSE_CSR ) { + if( getDataType()==DataType.MATRIX && dimsKnown(true) ) { double matrixPSize = OptimizerUtils.estimatePartitionedSizeExactSparsity(_dim1, _dim2, _rows_in_block, _cols_in_block, _nnz); double dataCache = SparkExecutionContext.getDataMemoryBudget(true, true); - serializedStorage = (MatrixBlock.evalSparseFormatInMemory(_dim1, _dim2, _nnz) - && matrixPSize > dataCache ); //sparse in-memory does not fit in agg mem + serializedStorage = MatrixBlock.evalSparseFormatInMemory(_dim1, _dim2, _nnz) + && matrixPSize > dataCache //sparse in-memory does not fit in agg mem + && (OptimizerUtils.getSparsity(_dim1, _dim2, _nnz) < MatrixBlock.ULTRA_SPARSITY_TURN_POINT + || !Checkpoint.CHECKPOINT_SPARSE_CSR ); //ultra-sparse or sparse w/o csr } else if( !dimsKnown(true) ) { setRequiresRecompile();
