Repository: systemml Updated Branches: refs/heads/master 8e1146e89 -> 52891d28e
[MINOR] Simplification internal matrix block slicing of row blocks Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/27fff38b Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/27fff38b Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/27fff38b Branch: refs/heads/master Commit: 27fff38bcb67fe90297063371a8608278d93e0be Parents: 8e1146e Author: Matthias Boehm <[email protected]> Authored: Sun Jun 10 13:22:35 2018 -0700 Committer: Matthias Boehm <[email protected]> Committed: Sun Jun 10 13:22:35 2018 -0700 ---------------------------------------------------------------------- .../controlprogram/paramserv/ParamservUtils.java | 2 +- .../parfor/DataPartitionerRemoteSparkMapper.java | 3 +-- .../instructions/mr/CumulativeOffsetInstruction.java | 2 +- .../instructions/spark/AppendGSPInstruction.java | 2 +- .../spark/CumulativeOffsetSPInstruction.java | 2 +- .../instructions/spark/utils/RDDConverterUtils.java | 2 +- .../runtime/instructions/spark/utils/RDDSortUtils.java | 12 ++++++------ .../sysml/runtime/matrix/data/LibMatrixReorg.java | 7 +++---- .../apache/sysml/runtime/matrix/data/MatrixBlock.java | 4 ++++ .../runtime/matrix/data/OperationsOnMatrixValues.java | 3 +-- 10 files changed, 20 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/27fff38b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java index 426a7fe..37b184f 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/paramserv/ParamservUtils.java @@ -90,7 +90,7 @@ public class ParamservUtils { new MetaDataFormat(new MatrixCharacteristics(-1, -1, -1, -1), OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo)); MatrixBlock tmp = mo.acquireRead(); - result.acquireModify(tmp.slice((int) rl - 1, (int) rh - 1, 0, tmp.getNumColumns() - 1, new MatrixBlock())); + result.acquireModify(tmp.slice((int) rl - 1, (int) rh - 1)); mo.release(); result.release(); result.enableCleanup(false); http://git-wip-us.apache.org/repos/asf/systemml/blob/27fff38b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java index 8e62f42..b5f0956 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java @@ -117,8 +117,7 @@ public class DataPartitionerRemoteSparkMapper extends ParWorker implements PairF for( int i=0; i<rows; i+=_n ) { PairWritableBlock tmp = new PairWritableBlock(); tmp.indexes = new MatrixIndexes(1, col_offset/_bclen+1); - tmp.block = value2.slice(i, Math.min(i+(int)_n-1, value2.getNumRows()-1), - 0, value2.getNumColumns()-1, new MatrixBlock()); + tmp.block = value2.slice(i, Math.min(i+(int)_n-1, value2.getNumRows()-1)); ret.add(new Tuple2<Long,Writable>(new Long((row_offset+i)/_n+1),tmp)); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/27fff38b/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeOffsetInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeOffsetInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeOffsetInstruction.java index 817ba31..899129f 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeOffsetInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/CumulativeOffsetInstruction.java @@ -83,7 +83,7 @@ public class CumulativeOffsetInstruction extends BinaryInstruction { //blockwise offset aggregation and prefix sum computation MatrixBlock data2 = new MatrixBlock(data); //cp data - MatrixBlock fdata2 = data2.slice(0, 0, 0, data2.getNumColumns()-1, new MatrixBlock()); //1-based + MatrixBlock fdata2 = data2.slice(0, 0); fdata2.binaryOperationsInPlace(_bop, offset); //sum offset to first row data2.copy(0, 0, 0, data2.getNumColumns()-1, fdata2, true); //0-based data2.unaryOperations(_uop, blk); //compute columnwise prefix sums/prod/min/max http://git-wip-us.apache.org/repos/asf/systemml/blob/27fff38b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGSPInstruction.java index 6c83387..3ecb91f 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGSPInstruction.java @@ -219,7 +219,7 @@ public class AppendGSPInstruction extends BinarySPInstruction { } else { // Since merge requires the dimensions matching, shifting = slicing + left indexing - MatrixBlock firstSlicedBlk = in.slice(0, cutAt-1, 0, in.getNumColumns()-1, new MatrixBlock()); + MatrixBlock firstSlicedBlk = in.slice(0, cutAt-1); MatrixBlock firstBlk = new MatrixBlock(lblen1, in.getNumColumns(), true); firstBlk = firstBlk.leftIndexingOperations(firstSlicedBlk, _shiftBy, _blen-1, 0, in.getNumColumns()-1, new MatrixBlock(), UpdateType.INPLACE_PINNED); http://git-wip-us.apache.org/repos/asf/systemml/blob/27fff38b/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeOffsetSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeOffsetSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeOffsetSPInstruction.java index 05b827f..6e151c6 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeOffsetSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeOffsetSPInstruction.java @@ -181,7 +181,7 @@ public class CumulativeOffsetSPInstruction extends BinarySPInstruction { //blockwise offset aggregation and prefix sum computation MatrixBlock data2 = new MatrixBlock(dblkIn); //cp data - MatrixBlock fdata2 = data2.slice(0, 0, 0, data2.getNumColumns()-1, new MatrixBlock()); //1-based + MatrixBlock fdata2 = data2.slice(0, 0); fdata2.binaryOperationsInPlace(_bop, oblkIn); //sum offset to first row data2.copy(0, 0, 0, data2.getNumColumns()-1, fdata2, true); //0-based data2.unaryOperations(_uop, blkOut); //compute columnwise prefix sums/prod/min/max http://git-wip-us.apache.org/repos/asf/systemml/blob/27fff38b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java index 29cd567..fe5aab1 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java @@ -952,7 +952,7 @@ public class RDDConverterUtils MatrixIndexes ix = arg0._1(); MatrixBlock blk = arg0._2(); for( int i=0; i<blk.getNumRows(); i++ ) { - MatrixBlock tmpBlk = blk.slice(i, i, 0, blk.getNumColumns()-1, new MatrixBlock()); + MatrixBlock tmpBlk = blk.slice(i, i); long rix = UtilFunctions.computeCellIndex(ix.getRowIndex(), _brlen, i); ret.add(new Tuple2<>(rix, new Tuple2<>(ix.getColumnIndex(),tmpBlk))); } http://git-wip-us.apache.org/repos/asf/systemml/blob/27fff38b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java index ef2bca3..01bdef8 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java @@ -291,7 +291,7 @@ public class RDDSortUtils throws Exception { ArrayList<MatrixBlock> rows = new ArrayList<>(); for(int i=0; i<arg0.getNumRows(); i++) - rows.add(arg0.slice(i, i, 0, arg0.getNumColumns()-1, new MatrixBlock())); + rows.add(arg0.slice(i, i)); return rows.iterator(); } } @@ -368,7 +368,7 @@ public class RDDSortUtils long ixoffset = (ix.getRowIndex()-1)*_brlen; for( int i=0; i<mb.getNumRows(); i++) { double[] vals = DataConverter.convertToDoubleVector( - mb.slice(i, i, 0, mb.getNumColumns()-1, new MatrixBlock())); + mb.slice(i, i)); ret.add(new Tuple2<>(new ValuesIndexPair(vals,ixoffset+i+1), vals)); } @@ -773,9 +773,9 @@ public class RDDSortUtils long valix = (long) mbTargetIndex.getValue(_currPos, 0); long rix = UtilFunctions.computeBlockIndex(valix, _brlen); int pos = UtilFunctions.computeCellInBlock(valix, _brlen); - int len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); + int len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); MatrixIndexes lix = new MatrixIndexes(rix,ixmap.getColumnIndex()); - MatrixBlock tmp = data.slice(_currPos, _currPos, 0, data.getNumColumns()-1, new MatrixBlock()); + MatrixBlock tmp = data.slice(_currPos, _currPos); _currPos++; //handle end of block situations @@ -862,9 +862,9 @@ public class RDDSortUtils long valix = (long) mbTargetIndex.getValue(_currPos, 0); long rix = UtilFunctions.computeBlockIndex(valix, _brlen); int pos = UtilFunctions.computeCellInBlock(valix, _brlen); - int len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); + int len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); MatrixIndexes lix = new MatrixIndexes(rix,ixmap.getColumnIndex()); - MatrixBlock tmp = data.slice(_currPos, _currPos, 0, data.getNumColumns()-1, new MatrixBlock()); + MatrixBlock tmp = data.slice(_currPos, _currPos); _currPos++; //handle end of block situations http://git-wip-us.apache.org/repos/asf/systemml/blob/27fff38b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java index ad4055b..2a35cf4 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java @@ -263,7 +263,7 @@ public class LibMatrixReorg //slice first block MatrixIndexes outix1 = new MatrixIndexes(blkix1, inix.getColumnIndex()); MatrixBlock outblk1 = new MatrixBlock(blklen1, inblk.getNumColumns(), inblk.isInSparseFormat()); - MatrixBlock tmp1 = tmpblk.slice(0, iposCut, 0, tmpblk.getNumColumns()-1, new MatrixBlock()); + MatrixBlock tmp1 = tmpblk.slice(0, iposCut); outblk1.leftIndexingOperations(tmp1, ipos1, ipos1+tmp1.getNumRows()-1, 0, tmpblk.getNumColumns()-1, outblk1, UpdateType.INPLACE_PINNED); out.add(new IndexedMatrixValue(outix1, outblk1)); @@ -272,7 +272,7 @@ public class LibMatrixReorg if( blkix1 != blkix2 ) { MatrixIndexes outix2 = new MatrixIndexes(blkix2, inix.getColumnIndex()); MatrixBlock outblk2 = new MatrixBlock(blklen2, inblk.getNumColumns(), inblk.isInSparseFormat()); - MatrixBlock tmp2 = tmpblk.slice(iposCut+1, tmpblk.getNumRows()-1, 0, tmpblk.getNumColumns()-1, new MatrixBlock()); + MatrixBlock tmp2 = tmpblk.slice(iposCut+1, tmpblk.getNumRows()-1); outblk2.leftIndexingOperations(tmp2, 0, tmp2.getNumRows()-1, 0, tmpblk.getNumColumns()-1, outblk2, UpdateType.INPLACE_PINNED); out.add(new IndexedMatrixValue(outix2, outblk2)); } @@ -670,8 +670,7 @@ public class LibMatrixReorg { for( int rl=0; rl<tmp.getNumRows(); rl+=brlen ) { MatrixBlock mb = tmp.slice( - rl, (int)(Math.min(rl+brlen, tmp.getNumRows())-1), - 0, tmp.getNumColumns()-1, new MatrixBlock()); + rl, (int)(Math.min(rl+brlen, tmp.getNumRows())-1)); outList.add(new IndexedMatrixValue( new MatrixIndexes(rl/brlen+1, ix.getRowIndex()), mb)); } http://git-wip-us.apache.org/repos/asf/systemml/blob/27fff38b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java index 5c285f8..393f8c7 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java @@ -3779,6 +3779,10 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab (int)ixrange.colStart, (int)ixrange.colEnd, true, ret); } + public MatrixBlock slice(int rl, int ru) { + return slice(rl, ru, 0, clen-1, true, new MatrixBlock()); + } + @Override public MatrixBlock slice(int rl, int ru, int cl, int cu, CacheBlock ret) { return slice(rl, ru, cl, cu, true, ret); http://git-wip-us.apache.org/repos/asf/systemml/blob/27fff38b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java index bc4e969..6866cf1 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java @@ -391,8 +391,7 @@ public class OperationsOnMatrixValues int row_offset = blockRow*brlen; //copy submatrix to block - MatrixBlock tmp = out.slice( row_offset, row_offset+maxRow-1, - 0, out.getNumColumns()-1, new MatrixBlock() ); + MatrixBlock tmp = out.slice(row_offset, row_offset+maxRow-1); //append block to result cache outlist.add(new IndexedMatrixValue(new MatrixIndexes(blockRow+1,ix.getColumnIndex()), tmp));
