[SYSTEMML-562][SYSTEMML-413] Basic cleanup of cache block abstraction Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/44c8f9d4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/44c8f9d4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/44c8f9d4
Branch: refs/heads/master Commit: 44c8f9d4b1120901f5bf9c14e686189cb17c948b Parents: 42906ba Author: Matthias Boehm <[email protected]> Authored: Wed Jul 27 22:52:22 2016 -0700 Committer: Matthias Boehm <[email protected]> Committed: Wed Jul 27 22:52:22 2016 -0700 ---------------------------------------------------------------------- .../controlprogram/caching/CacheBlock.java | 49 ++++++++++++++------ .../caching/CacheBlockFactory.java | 18 +++++++ .../spark/data/PartitionedBlock.java | 11 ++--- .../sysml/runtime/matrix/data/FrameBlock.java | 13 ------ .../sysml/runtime/matrix/data/MatrixBlock.java | 11 ----- .../matrix/data/OperationsOnMatrixValues.java | 24 ++++++++++ 6 files changed, 81 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/44c8f9d4/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheBlock.java index f58a7c2..b73584c 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheBlock.java @@ -19,12 +19,8 @@ package org.apache.sysml.runtime.controlprogram.caching; -import java.util.ArrayList; - import org.apache.hadoop.io.Writable; import org.apache.sysml.runtime.DMLRuntimeException; -import org.apache.sysml.runtime.matrix.data.Pair; -import org.apache.sysml.runtime.util.IndexRange; /** @@ -35,6 +31,18 @@ import org.apache.sysml.runtime.util.IndexRange; public interface CacheBlock extends Writable { /** + * + * @return + */ + public int getNumRows(); + + /** + * + * @return + */ + public int getNumColumns(); + + /** * Get the in-memory size in bytes of the cache block. * @return */ @@ -60,18 +68,29 @@ public interface CacheBlock extends Writable */ public void compactEmptyBlock(); - public int getNumRows(); - public int getNumColumns(); - + /** + * Slice a sub block out of the current block and write into the given output block. + * This method returns the passed instance if not null. + * + * @param rl + * @param ru + * @param cl + * @param cu + * @param block + * @return + * @throws DMLRuntimeException + */ public CacheBlock sliceOperations(int rl, int ru, int cl, int cu, CacheBlock block) - throws DMLRuntimeException; + throws DMLRuntimeException; + /** + * Merge the given block into the current block. Both blocks needs to be of equal + * dimensions and contain disjoint non-zero cells. + * + * @param that + * @param appendOnly + * @throws DMLRuntimeException + */ public void merge(CacheBlock that, boolean appendOnly) - throws DMLRuntimeException; - - @SuppressWarnings("rawtypes") - public ArrayList getPairList(); - - ArrayList<Pair<?, ?>> performSlice(IndexRange ixrange, int brlen, int bclen, int iix, int jix, CacheBlock in) - throws DMLRuntimeException; + throws DMLRuntimeException; } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/44c8f9d4/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheBlockFactory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheBlockFactory.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheBlockFactory.java index 3bf86b5..9595441 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheBlockFactory.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheBlockFactory.java @@ -19,8 +19,12 @@ package org.apache.sysml.runtime.controlprogram.caching; +import java.util.ArrayList; + import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.MatrixBlock; +import org.apache.sysml.runtime.matrix.data.MatrixIndexes; +import org.apache.sysml.runtime.matrix.data.Pair; /** * Factory to create instances of matrix/frame blocks given @@ -54,4 +58,18 @@ public class CacheBlockFactory return 1; throw new RuntimeException("Unsupported cache block type: "+block.getClass().getName()); } + + /** + * + * @param block + * @return + */ + public static ArrayList<?> getPairList(CacheBlock block) { + int code = getCode(block); + switch( code ) { + case 0: return new ArrayList<Pair<MatrixIndexes,MatrixBlock>>(); + case 1: return new ArrayList<Pair<Long,FrameBlock>>(); + } + throw new RuntimeException("Unsupported cache block type: "+code); + } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/44c8f9d4/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java index 465fdd5..fbd7a25 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.controlprogram.caching.CacheBlock; import org.apache.sysml.runtime.controlprogram.caching.CacheBlockFactory; +import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues; import org.apache.sysml.runtime.matrix.data.Pair; import org.apache.sysml.runtime.util.FastBufferedDataInputStream; import org.apache.sysml.runtime.util.FastBufferedDataOutputStream; @@ -260,19 +261,17 @@ public class PartitionedBlock<T extends CacheBlock> implements Externalizable int lcl = (int) cl; int lcu = (int) cu; - ArrayList<Pair<?, ?>> allBlks = block.getPairList(); + ArrayList<Pair<?, ?>> allBlks = (ArrayList<Pair<?, ?>>) CacheBlockFactory.getPairList(block); int start_iix = (lrl-1)/_brlen+1; int end_iix = (lru-1)/_brlen+1; int start_jix = (lcl-1)/_bclen+1; int end_jix = (lcu-1)/_bclen+1; for( int iix = start_iix; iix <= end_iix; iix++ ) - for(int jix = start_jix; jix <= end_jix; jix++) - { - T in = getBlock(iix, jix); + for(int jix = start_jix; jix <= end_jix; jix++) { IndexRange ixrange = new IndexRange(rl, ru, cl, cu); - ArrayList<Pair<?, ?>> outlist = block.performSlice(ixrange, _brlen, _bclen, iix, jix, in); - allBlks.addAll(outlist); + allBlks.addAll(OperationsOnMatrixValues.performSlice( + ixrange, _brlen, _bclen, iix, jix, getBlock(iix, jix))); } if(allBlks.size() == 1) { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/44c8f9d4/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java index 78fa165..1c8ffd0 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java @@ -1305,17 +1305,4 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable _mvValue = mvVal; } } - - //TODO generalize these methods and remove from frame block - - @Override - public ArrayList getPairList() { - return new ArrayList<Pair<Long, FrameBlock>>(); - } - - public ArrayList<Pair<?, ?>> performSlice(IndexRange ixrange, int brlen, int bclen, int iix, int jix, CacheBlock in) throws DMLRuntimeException - { - return OperationsOnMatrixValues.performSlice(ixrange, brlen, bclen, iix, jix, (FrameBlock)in); - } - } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/44c8f9d4/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java index 842982d..452eddb 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java @@ -6240,15 +6240,4 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab } public SparsityEstimate(){} } - - public ArrayList<Pair<MatrixIndexes, MatrixBlock>> getPairList() - { - return new ArrayList<Pair<MatrixIndexes,MatrixBlock>>(); - } - - @SuppressWarnings("unchecked") - public ArrayList<Pair<?, ?>> performSlice(IndexRange ixrange, int brlen, int bclen, int iix, int jix, CacheBlock in) throws DMLRuntimeException - { - return OperationsOnMatrixValues.performSlice(ixrange, brlen, bclen, iix, jix, (MatrixBlock)in); - } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/44c8f9d4/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java index 219a1e7..a105e6d 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java @@ -27,6 +27,7 @@ import java.util.List; import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.compress.CompressedMatrixBlock; +import org.apache.sysml.runtime.controlprogram.caching.CacheBlock; import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType; import org.apache.sysml.runtime.functionobjects.Builtin; import org.apache.sysml.runtime.instructions.spark.utils.SparkUtils; @@ -300,6 +301,29 @@ public class OperationsOnMatrixValues value1.aggregateBinaryOperations(value1, value2, valueOut, op); } + /** + * + * @param ixrange + * @param brlen + * @param bclen + * @param iix + * @param jix + * @param in + * @return + * @throws DMLRuntimeException + */ + @SuppressWarnings("rawtypes") + public static ArrayList performSlice(IndexRange ixrange, int brlen, int bclen, int iix, int jix, CacheBlock in) + throws DMLRuntimeException + { + if( in instanceof MatrixBlock ) + return performSlice(ixrange, brlen, bclen, iix, jix, (MatrixBlock)in); + else if( in instanceof FrameBlock ) + return performSlice(ixrange, brlen, bclen, iix, jix, (FrameBlock)in); + throw new DMLRuntimeException("Unsupported cache block type: "+in.getClass().getName()); + } + + @SuppressWarnings("rawtypes") public static ArrayList performSlice(IndexRange ixrange, int brlen, int bclen, int iix, int jix, MatrixBlock in) throws DMLRuntimeException
