[MINOR] Improved primitives for memory profiling and sparse checks Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/f13f697e Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/f13f697e Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/f13f697e
Branch: refs/heads/master Commit: f13f697e08a2a87cff61ca5dfd62f73f10555d3e Parents: f84fd71 Author: Matthias Boehm <[email protected]> Authored: Wed Dec 6 19:36:54 2017 -0800 Committer: Matthias Boehm <[email protected]> Committed: Thu Dec 7 13:36:08 2017 -0800 ---------------------------------------------------------------------- .../sysml/runtime/controlprogram/LocalVariableMap.java | 9 +++++++++ .../apache/sysml/runtime/controlprogram/ProgramBlock.java | 2 +- .../sysml/runtime/controlprogram/caching/CacheableData.java | 6 +++++- .../org/apache/sysml/runtime/matrix/data/MatrixBlock.java | 4 ++++ 4 files changed, 19 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/f13f697e/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java index e894495..c91eec4 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java @@ -27,6 +27,7 @@ import java.util.Set; import java.util.StringTokenizer; import org.apache.sysml.runtime.DMLRuntimeException; +import org.apache.sysml.runtime.controlprogram.caching.CacheableData; import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter; import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence; import org.apache.sysml.runtime.instructions.cp.Data; @@ -122,6 +123,14 @@ public class LocalVariableMap implements Cloneable return outputs; } + public double getPinnedDataSize() { + //note: this method returns the total size of pinned data objects + //that are not subject to automatic eviction. + return localMap.values().stream() + .filter(d -> (d instanceof CacheableData)) + .mapToDouble(d -> ((CacheableData<?>)d).getDataSize()).sum(); + } + public String serialize() throws DMLRuntimeException { StringBuilder sb = new StringBuilder(); int count = 0; http://git-wip-us.apache.org/repos/asf/systemml/blob/f13f697e/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java index fd6909d..0996cd9 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java @@ -371,7 +371,7 @@ public class ProgramBlock implements ParseInfo if( nnz1 != nnz2 ) throw new DMLRuntimeException("Matrix nnz meta data was incorrect: ("+varname+", actual="+nnz1+", expected="+nnz2+", inst="+lastInst+")"); - if( sparse1 != sparse2 ) + if( sparse1 != sparse2 && mb.isAllocated() ) throw new DMLRuntimeException("Matrix was in wrong data representation: ("+varname+", actual="+sparse1+", expected="+sparse2 + ", nrow="+mb.getNumRows()+", ncol="+mb.getNumColumns()+", nnz="+nnz1+", inst="+lastInst+")"); } http://git-wip-us.apache.org/repos/asf/systemml/blob/f13f697e/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java index be7b9b9..e2b9838 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java @@ -956,6 +956,10 @@ public abstract class CacheableData<T extends CacheBlock> extends Data return (_data.getInMemorySize() <= CACHING_THRESHOLD); } + public long getDataSize() { + return (_data != null) ?_data.getInMemorySize() : 0; + } + protected ValueType[] getSchema() { return null; } @@ -1262,7 +1266,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data sizePinned.set( Math.max(size,0) ); } - protected long getPinnedSize() { + protected static long getPinnedSize() { return sizePinned.get(); } http://git-wip-us.apache.org/repos/asf/systemml/blob/f13f697e/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java index c893bc6..ac0655c 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java @@ -1312,9 +1312,13 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab int apos = sparseBlock.pos(i); int alen = sparseBlock.size(i); int[] aix = sparseBlock.indexes(i); + double[] avals = sparseBlock.values(i); for( int k=apos+1; k<apos+alen; k++ ) if( aix[k-1] >= aix[k] ) throw new RuntimeException("Wrong sparse row ordering: "+k+" "+aix[k-1]+" "+aix[k]); + for( int k=apos; k<apos+alen; k++ ) + if( avals[k] == 0 ) + throw new RuntimeException("Wrong sparse row: zero at "+k); } }
