Repository: systemml Updated Branches: refs/heads/master 62a1b75ba -> 8707ff1d4
[SYSTEMML-1762] Fix spark/mr reshape instructions (missing blocks) This patch fixes special cases of spark/mr reshape instructions, which applies to sparse and dense. Given an input block, we compute the first and last output block index and create all blocks in between. In cases where we output at least three blocks and the first and last block have the same row index (e.g., (1,12) and (1,1)), we missed to create the remaining blocks after the first till the end of the row block (e.g., the blocks (1,13) and (1,14) if the matrix has 14 column blocks), leading to null pointer exceptions on the actual reshape copy. Furthermore, this patch also cleans up the respective sparse and dense reshape interfaces and implementations as well as adds tests for the encountered special cases. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/8707ff1d Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/8707ff1d Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/8707ff1d Branch: refs/heads/master Commit: 8707ff1d41cdb9ba308d1fc22f3d44155eb8ab88 Parents: 62a1b75 Author: Matthias Boehm <mboe...@gmail.com> Authored: Thu Jul 13 00:48:17 2017 -0700 Committer: Matthias Boehm <mboe...@gmail.com> Committed: Thu Jul 13 00:48:17 2017 -0700 ---------------------------------------------------------------------- .../mr/MatrixReshapeMRInstruction.java | 22 +- .../spark/MatrixReshapeSPInstruction.java | 3 +- .../runtime/matrix/data/LibMatrixReorg.java | 203 ++++++++----------- .../runtime/matrix/data/MatrixIndexes.java | 6 +- .../functions/reorg/VectorReshapeTest.java | 130 ++++++++++++ .../scripts/functions/reorg/VectorReshape.R | 29 +++ .../scripts/functions/reorg/VectorReshape.dml | 25 +++ .../functions/reorg/ZPackageSuite.java | 3 +- 8 files changed, 280 insertions(+), 141 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/8707ff1d/src/main/java/org/apache/sysml/runtime/instructions/mr/MatrixReshapeMRInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/MatrixReshapeMRInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/MatrixReshapeMRInstruction.java index 2662a5c..4844a6e 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/mr/MatrixReshapeMRInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/MatrixReshapeMRInstruction.java @@ -33,12 +33,10 @@ import org.apache.sysml.runtime.util.UtilFunctions; public class MatrixReshapeMRInstruction extends UnaryInstruction { - - private long _rows = -1; - private long _cols = -1; private boolean _byrow = false; private MatrixCharacteristics _mcIn = null; + private MatrixCharacteristics _mcOut = null; //MB: cache should be integrated with tempValues, but for n blocks private ArrayList<IndexedMatrixValue> _cache = null; @@ -48,9 +46,7 @@ public class MatrixReshapeMRInstruction extends UnaryInstruction super(op, in, out, istr); mrtype = MRINSTRUCTION_TYPE.MMTSJ; instString = istr; - - _rows = rows; - _cols = cols; + _mcOut = new MatrixCharacteristics(rows, cols, -1, -1); _byrow = byrow; } @@ -95,8 +91,8 @@ public class MatrixReshapeMRInstruction extends UnaryInstruction ArrayList<IndexedMatrixValue> out = _cache; //process instruction - out = LibMatrixReorg.reshape(imv, _mcIn.getRows(), _mcIn.getCols(), brlen, bclen, - out, _rows, _cols, brlen, bclen, _byrow); + _mcOut.setBlockSize(brlen, bclen); + out = LibMatrixReorg.reshape(imv, _mcIn, out, _mcOut, _byrow); //put the output values in the output cache for( IndexedMatrixValue outBlk : out ) @@ -108,13 +104,11 @@ public class MatrixReshapeMRInstruction extends UnaryInstruction } } - public long getNumRows() - { - return _rows; + public long getNumRows() { + return _mcOut.getRows(); } - public long getNumColunms() - { - return _cols; + public long getNumColunms() { + return _mcOut.getCols(); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/8707ff1d/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixReshapeSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixReshapeSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixReshapeSPInstruction.java index 5941285..f937c8e 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixReshapeSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixReshapeSPInstruction.java @@ -135,8 +135,7 @@ public class MatrixReshapeSPInstruction extends UnarySPInstruction //execute actual reshape operation ArrayList<IndexedMatrixValue> out = new ArrayList<IndexedMatrixValue>(); - out = LibMatrixReorg.reshape(in, _mcIn.getRows(), _mcIn.getCols(), _mcIn.getRowsPerBlock(), _mcIn.getRowsPerBlock(), - out, _mcOut.getRows(), _mcOut.getCols(), _mcOut.getRowsPerBlock(), _mcOut.getColsPerBlock(), _byrow); + out = LibMatrixReorg.reshape(in, _mcIn, out, _mcOut, _byrow); //output conversion (for compatibility w/ rdd schema) return SparkUtils.fromIndexedMatrixBlock(out).iterator(); http://git-wip-us.apache.org/repos/asf/systemml/blob/8707ff1d/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java index 205f787..cb98aee 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java @@ -39,6 +39,7 @@ import org.apache.sysml.runtime.functionobjects.DiagIndex; import org.apache.sysml.runtime.functionobjects.RevIndex; import org.apache.sysml.runtime.functionobjects.SortIndex; import org.apache.sysml.runtime.functionobjects.SwapIndex; +import org.apache.sysml.runtime.matrix.MatrixCharacteristics; import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue; import org.apache.sysml.runtime.matrix.operators.ReorgOperator; import org.apache.sysml.runtime.util.DataConverter; @@ -468,25 +469,19 @@ public class LibMatrixReorg /** - * MR reshape interface - for reshape we cannot view blocks independently, and hence, + * MR/SPARK reshape interface - for reshape we cannot view blocks independently, and hence, * there are different CP and MR interfaces. * * @param in indexed matrix value - * @param rows1 number of rows 1 - * @param cols1 number of columns 1 - * @param brlen1 number of rows in a block 1 - * @param bclen1 number of columns in a block 1 + * @param mcIn input matrix characteristics * @param out list of indexed matrix values - * @param rows2 number of rows 2 - * @param cols2 number of columns 2 - * @param brlen2 number of rows in a block 2 - * @param bclen2 number of columns in a block 2 + * @param mcOut output matrix characteristics * @param rowwise if true, reshape by row * @return list of indexed matrix values * @throws DMLRuntimeException if DMLRuntimeException occurs */ - public static ArrayList<IndexedMatrixValue> reshape( IndexedMatrixValue in, long rows1, long cols1, int brlen1, int bclen1, - ArrayList<IndexedMatrixValue> out, long rows2, long cols2, int brlen2, int bclen2, boolean rowwise ) + public static ArrayList<IndexedMatrixValue> reshape( IndexedMatrixValue in, MatrixCharacteristics mcIn, + ArrayList<IndexedMatrixValue> out, MatrixCharacteristics mcOut, boolean rowwise ) throws DMLRuntimeException { //prepare inputs @@ -494,16 +489,16 @@ public class LibMatrixReorg MatrixBlock mbIn = (MatrixBlock) in.getValue(); //prepare result blocks (no reuse in order to guarantee mem constraints) - Collection<MatrixIndexes> rix = computeAllResultBlockIndexes(ixIn, rows1, cols1, brlen1, bclen1, rows2, cols2, brlen2, bclen2, rowwise); - HashMap<MatrixIndexes, MatrixBlock> rblk = createAllResultBlocks(rix, mbIn.nonZeros, rows1, cols1, brlen1, bclen1, rows2, cols2, brlen2, bclen2, rowwise, out); + Collection<MatrixIndexes> rix = computeAllResultBlockIndexes(ixIn, mcIn, mcOut, rowwise); + HashMap<MatrixIndexes, MatrixBlock> rblk = createAllResultBlocks(rix, mbIn.nonZeros, mcIn, mcOut, rowwise, out); //basic algorithm - long row_offset = (ixIn.getRowIndex()-1)*brlen1; - long col_offset = (ixIn.getColumnIndex()-1)*bclen1; + long row_offset = (ixIn.getRowIndex()-1)*mcIn.getRowsPerBlock(); + long col_offset = (ixIn.getColumnIndex()-1)*mcIn.getColsPerBlock(); if( mbIn.sparse ) - reshapeSparse(mbIn, row_offset, col_offset, rblk, rows1, cols1, rows2, cols2, brlen2, bclen2, rowwise); + reshapeSparse(mbIn, row_offset, col_offset, rblk, mcIn, mcOut, rowwise); else //dense - reshapeDense(mbIn, row_offset, col_offset, rblk, rows1, cols1, rows2, cols2, brlen2, bclen2, rowwise); + reshapeDense(mbIn, row_offset, col_offset, rblk, mcIn, mcOut, rowwise); //prepare output out = new ArrayList<IndexedMatrixValue>(); @@ -1409,22 +1404,18 @@ public class LibMatrixReorg /////////////////////////////// private static Collection<MatrixIndexes> computeAllResultBlockIndexes( MatrixIndexes ixin, - long rows1, long cols1, int brlen1, int bclen1, - long rows2, long cols2, int brlen2, int bclen2, boolean rowwise ) + MatrixCharacteristics mcIn, MatrixCharacteristics mcOut, boolean rowwise ) { HashSet<MatrixIndexes> ret = new HashSet<MatrixIndexes>(); - long nrblk2 = rows2/brlen2 + ((rows2%brlen2!=0)?1:0); - long ncblk2 = cols2/bclen2 + ((cols2%bclen2!=0)?1:0); + long row_offset = (ixin.getRowIndex()-1)*mcOut.getRowsPerBlock(); + long col_offset = (ixin.getColumnIndex()-1)*mcOut.getColsPerBlock(); - long row_offset = (ixin.getRowIndex()-1)*brlen1; - long col_offset = (ixin.getColumnIndex()-1)*bclen1; - - if( rowwise ){ - for( long i=row_offset; i<Math.min(rows1,row_offset+brlen1); i++ ) + if( rowwise ) { + for( long i=row_offset; i<Math.min(mcIn.getRows(),row_offset+mcIn.getRowsPerBlock()); i++ ) { - MatrixIndexes first = computeResultBlockIndex(new MatrixIndexes(), i, col_offset, rows1, cols1, rows2, cols2, brlen2, bclen2, rowwise); - MatrixIndexes last = computeResultBlockIndex(new MatrixIndexes(), i, Math.min(cols1,col_offset+bclen1)-1, rows1, cols1, rows2, cols2, brlen2, bclen2, rowwise); + MatrixIndexes first = computeResultBlockIndex(new MatrixIndexes(), i, col_offset, mcIn, mcOut, rowwise); + MatrixIndexes last = computeResultBlockIndex(new MatrixIndexes(), i, Math.min(mcIn.getCols(),col_offset+mcIn.getColsPerBlock())-1, mcIn, mcOut, rowwise); if( first.getRowIndex()<=0 || first.getColumnIndex()<=0 ) throw new RuntimeException("Invalid computed first index: "+first.toString()); @@ -1435,26 +1426,23 @@ public class LibMatrixReorg ret.add(first); //add blocks in between first and last - for( long k1=first.getRowIndex(); k1<=last.getRowIndex(); k1++ ) - { - long k2_start = ((k1==first.getRowIndex()) ? first.getColumnIndex()+1 : 1); - long k2_end = ((k1==last.getRowIndex()) ? last.getColumnIndex()-1 : ncblk2); - - for( long k2=k2_start; k2<=k2_end; k2++ ) { - ret.add(new MatrixIndexes(k1,k2)); + if( !first.equals(last) ) { + for( long k1=first.getRowIndex(); k1<=last.getRowIndex(); k1++ ) { + long k2_start = ((k1==first.getRowIndex()) ? first.getColumnIndex()+1 : 1); + long k2_end = ((k1==last.getRowIndex() && first.getRowIndex()!=last.getRowIndex()) ? + last.getColumnIndex()-1 : mcOut.getNumColBlocks()); + for( long k2=k2_start; k2<=k2_end; k2++ ) + ret.add(new MatrixIndexes(k1,k2)); } + ret.add(last); } - - //add last row block - ret.add(last); - } } else{ //colwise - for( long j=col_offset; j<Math.min(cols1,col_offset+bclen1); j++ ) + for( long j=col_offset; j<Math.min(mcIn.getCols(),col_offset+mcIn.getColsPerBlock()); j++ ) { - MatrixIndexes first = computeResultBlockIndex(new MatrixIndexes(), row_offset, j, rows1, cols1, rows2, cols2, brlen2, bclen2, rowwise); - MatrixIndexes last = computeResultBlockIndex(new MatrixIndexes(), Math.min(rows1,row_offset+brlen1)-1, j, rows1, cols1, rows2, cols2, brlen2, bclen2, rowwise); + MatrixIndexes first = computeResultBlockIndex(new MatrixIndexes(), row_offset, j, mcIn, mcOut, rowwise); + MatrixIndexes last = computeResultBlockIndex(new MatrixIndexes(), Math.min(mcIn.getRows(),row_offset+mcIn.getRowsPerBlock())-1, j, mcIn, mcOut, rowwise); if( first.getRowIndex()<=0 || first.getColumnIndex()<=0 ) throw new RuntimeException("Invalid computed first index: "+first.toString()); @@ -1463,19 +1451,18 @@ public class LibMatrixReorg //add first row block ret.add(first); + //add blocks in between first and last - for( long k1=first.getColumnIndex(); k1<=last.getColumnIndex(); k1++ ) - { - long k2_start = ((k1==first.getColumnIndex()) ? first.getRowIndex()+1 : 1); - long k2_end = ((k1==last.getColumnIndex()) ? last.getRowIndex()-1 : nrblk2); - - for( long k2=k2_start; k2<=k2_end; k2++ ) { - ret.add(new MatrixIndexes(k1,k2)); + if( !first.equals(last) ) { + for( long k1=first.getColumnIndex(); k1<=last.getColumnIndex(); k1++ ) { + long k2_start = ((k1==first.getColumnIndex()) ? first.getRowIndex()+1 : 1); + long k2_end = ((k1==last.getColumnIndex() && first.getRowIndex()!=last.getRowIndex()) ? + last.getRowIndex()-1 : mcOut.getNumRowBlocks()); + for( long k2=k2_start; k2<=k2_end; k2++ ) + ret.add(new MatrixIndexes(k1,k2)); } + ret.add(last); } - - //add last row block - ret.add(last); } } @@ -1483,23 +1470,21 @@ public class LibMatrixReorg } @SuppressWarnings("unused") - private static HashMap<MatrixIndexes, MatrixBlock> createAllResultBlocks( Collection<MatrixIndexes> rix, - long nnz, long rows1, long cols1, int brlen1, int bclen1, - long rows2, long cols2, int brlen2, int bclen2, boolean rowwise, ArrayList<IndexedMatrixValue> reuse ) + private static HashMap<MatrixIndexes, MatrixBlock> createAllResultBlocks( Collection<MatrixIndexes> rix, + long nnz, MatrixCharacteristics mcIn, MatrixCharacteristics mcOut, + boolean rowwise, ArrayList<IndexedMatrixValue> reuse ) { HashMap<MatrixIndexes, MatrixBlock> ret = new HashMap<MatrixIndexes,MatrixBlock>(); long nBlocks = rix.size(); int count = 0; - //System.out.println("Reuse "+((reuse!=null)?reuse.size():0)+"/"+nBlocks); - for( MatrixIndexes ix : rix ) { //compute indexes long bi = ix.getRowIndex(); long bj = ix.getColumnIndex(); - int lbrlen = (int) Math.min(brlen2, rows2-(bi-1)*brlen2); - int lbclen = (int) Math.min(bclen2, cols2-(bj-1)*bclen2); + int lbrlen = UtilFunctions.computeBlockSize(mcOut.getRows(), bi, mcOut.getRowsPerBlock()); + int lbclen = UtilFunctions.computeBlockSize(mcOut.getCols(), bj, mcOut.getColsPerBlock()); //create result block int estnnz = (int) (nnz/nBlocks); //force initialcapacity per row to 1, for many blocks @@ -1512,7 +1497,6 @@ public class LibMatrixReorg else block = new MatrixBlock(lbrlen, lbclen, sparse, estnnz); - //System.out.println("create block ("+bi+","+bj+"): "+lbrlen+" "+lbclen); if( lbrlen<1 || lbclen<1 ) throw new RuntimeException("Computed block dimensions ("+bi+","+bj+" -> "+lbrlen+","+lbclen+") are invalid!"); @@ -1524,8 +1508,8 @@ public class LibMatrixReorg private static void reshapeDense( MatrixBlock in, long row_offset, long col_offset, HashMap<MatrixIndexes,MatrixBlock> rix, - long rows1, long cols1, - long rows2, long cols2, int brlen2, int bclen2, boolean rowwise ) + MatrixCharacteristics mcIn, MatrixCharacteristics mcOut, boolean rowwise ) + throws DMLRuntimeException { if( in.isEmptyBlock(false) ) return; @@ -1544,17 +1528,18 @@ public class LibMatrixReorg double val = a[ aix+j ]; if( val !=0 ) { long aj = col_offset+j; - computeResultBlockIndex(ixtmp, ai, aj, rows1, cols1, rows2, cols2, brlen2, bclen2, rowwise); + ixtmp = computeResultBlockIndex(ixtmp, ai, aj, mcIn, mcOut, rowwise); MatrixBlock out = rix.get(ixtmp); - computeInBlockIndex(ixtmp, ai, aj, rows1, cols1, rows2, cols2, brlen2, bclen2, rowwise); + if( out == null ) + throw new DMLRuntimeException("Missing result block: "+ixtmp); + ixtmp = computeInBlockIndex(ixtmp, ai, aj, mcIn, mcOut, rowwise); out.appendValue((int)ixtmp.getRowIndex(),(int)ixtmp.getColumnIndex(), val); } } } //cleanup for sparse blocks - if( !rowwise ) - { + if( !rowwise ) { for( MatrixBlock block : rix.values() ) if( block.sparse ) block.sortSparseRows(); @@ -1562,9 +1547,9 @@ public class LibMatrixReorg } private static void reshapeSparse( MatrixBlock in, long row_offset, long col_offset, - HashMap<MatrixIndexes,MatrixBlock> rix, - long rows1, long cols1, - long rows2, long cols2, int brlen2, int bclen2, boolean rowwise ) + HashMap<MatrixIndexes,MatrixBlock> rix, + MatrixCharacteristics mcIn, MatrixCharacteristics mcOut, boolean rowwise ) + throws DMLRuntimeException { if( in.isEmptyBlock(false) ) return; @@ -1585,17 +1570,18 @@ public class LibMatrixReorg for( int j=apos; j<apos+alen; j++ ) { long aj = col_offset+aix[j]; - computeResultBlockIndex(ixtmp, ai, aj, rows1, cols1, rows2, cols2, brlen2, bclen2, rowwise); + ixtmp = computeResultBlockIndex(ixtmp, ai, aj, mcIn, mcOut, rowwise); MatrixBlock out = rix.get(ixtmp); - computeInBlockIndex(ixtmp, ai, aj, rows1, cols1, rows2, cols2, brlen2, bclen2, rowwise); + if( out == null ) + throw new DMLRuntimeException("Missing result block: "+ixtmp); + ixtmp = computeInBlockIndex(ixtmp, ai, aj, mcIn, mcOut, rowwise); out.appendValue((int)ixtmp.getRowIndex(),(int)ixtmp.getColumnIndex(), avals[j]); } } } //cleanup for sparse blocks - if( !rowwise ) - { + if( !rowwise ) { for( MatrixBlock block : rix.values() ) if( block.sparse ) block.sortSparseRows(); @@ -1605,63 +1591,36 @@ public class LibMatrixReorg /** * Assumes internal (0-begin) indices ai, aj as input; computes external block indexes (1-begin) * - * @param ixout matrix indexes - * @param ai ? - * @param aj ? - * @param rows1 ? - * @param cols1 ? - * @param rows2 ? - * @param cols2 ? - * @param brlen2 ? - * @param bclen2 ? - * @param rowwise ? + * @param ixout matrix indexes for reuse + * @param ai row index + * @param aj column index + * @param mcIn input matrix characteristics + * @param mcOut output matrix characteristics + * @param rowwise row-wise extraction * @return matrix indexes */ private static MatrixIndexes computeResultBlockIndex( MatrixIndexes ixout, long ai, long aj, - long rows1, long cols1, long rows2, long cols2, int brlen2, int bclen2, boolean rowwise ) + MatrixCharacteristics mcIn, MatrixCharacteristics mcOut, boolean rowwise ) { - long ci, cj, tempc; - long bci, bcj; - - if( rowwise ) { - tempc = ai*cols1+aj; - ci = tempc/cols2; - cj = tempc%cols2; - bci = ci/brlen2 + 1; - bcj = cj/bclen2 + 1; - } - else { //colwise - tempc = ai+rows1*aj; - ci = tempc%rows2; - cj = tempc/rows2; - bci = ci/brlen2 + 1; - bcj = cj/bclen2 + 1; - } - - ixout.setIndexes(bci, bcj); - return ixout; + long tempc = rowwise ? ai*mcIn.getCols()+aj : ai+mcIn.getRows()*aj; + long ci = rowwise ? tempc/mcOut.getCols() : tempc%mcOut.getRows(); + long cj = rowwise ? tempc%mcOut.getCols() : tempc/mcOut.getRows(); + long bci = ci/mcOut.getRowsPerBlock() + 1; + long bcj = cj/mcOut.getColsPerBlock() + 1; + return (ixout != null) ? ixout.setIndexes(bci, bcj) : + new MatrixIndexes(bci, bcj); } private static MatrixIndexes computeInBlockIndex( MatrixIndexes ixout, long ai, long aj, - long rows1, long cols1, long rows2, long cols2, int brlen2, int bclen2, boolean rowwise ) + MatrixCharacteristics mcIn, MatrixCharacteristics mcOut, boolean rowwise ) { - long ci, cj, tempc; - - if( rowwise ) { - tempc = ai*cols1+aj; - ci = (tempc/cols2)%brlen2; - cj = (tempc%cols2)%bclen2; - } - else { //colwise - tempc = ai+rows1*aj; - ci = (tempc%rows2)%brlen2; - cj = (tempc/rows2)%bclen2; - } - - //System.out.println("ai/aj: "+ai+"/"+aj+" -> ci/cj: "+ci+"/"+cj); - - ixout.setIndexes(ci, cj); - return ixout; + long tempc = rowwise ? ai*mcIn.getCols()+aj : ai+mcIn.getRows()*aj; + long ci = rowwise ? (tempc/mcOut.getCols())%mcOut.getRowsPerBlock() : + (tempc%mcOut.getRows())%mcOut.getRowsPerBlock(); + long cj = rowwise ? (tempc%mcOut.getCols())%mcOut.getColsPerBlock() : + (tempc/mcOut.getRows())%mcOut.getColsPerBlock(); + return (ixout != null) ? ixout.setIndexes(ci, cj) : + new MatrixIndexes(ci, cj); } private static MatrixBlock removeEmptyRows(MatrixBlock in, MatrixBlock ret, MatrixBlock select) http://git-wip-us.apache.org/repos/asf/systemml/blob/8707ff1d/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixIndexes.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixIndexes.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixIndexes.java index 3f52a5e..5dba256 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixIndexes.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixIndexes.java @@ -70,14 +70,16 @@ public class MatrixIndexes implements WritableComparable<MatrixIndexes>, RawComp return _col; } - public void setIndexes(long r, long c) { + public MatrixIndexes setIndexes(long r, long c) { _row = r; _col = c; + return this; } - public void setIndexes(MatrixIndexes that) { + public MatrixIndexes setIndexes(MatrixIndexes that) { _row = that._row; _col = that._col; + return this; } @Override http://git-wip-us.apache.org/repos/asf/systemml/blob/8707ff1d/src/test/java/org/apache/sysml/test/integration/functions/reorg/VectorReshapeTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/reorg/VectorReshapeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/reorg/VectorReshapeTest.java new file mode 100644 index 0000000..0be3e2f --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/reorg/VectorReshapeTest.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.reorg; + +import java.util.HashMap; + +import org.junit.Test; + +import org.apache.sysml.api.DMLScript; +import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.lops.LopProperties.ExecType; +import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.integration.TestConfiguration; +import org.apache.sysml.test.utils.TestUtils; + +public class VectorReshapeTest extends AutomatedTestBase +{ + private final static String TEST_NAME = "VectorReshape"; + private final static String TEST_DIR = "functions/reorg/"; + private static final String TEST_CLASS_DIR = TEST_DIR + VectorReshapeTest.class.getSimpleName() + "/"; + + private final static int rows1 = 1; + private final static int cols1 = 802816; + private final static int rows2 = 64; + private final static int cols2 = 12544; + + private final static double sparsityDense = 0.9; + private final static double sparsitySparse = 0.1; + + @Override + public void setUp() { + addTestConfiguration(TEST_NAME, new TestConfiguration( + TEST_CLASS_DIR, TEST_NAME, new String[] { "R" }) ); + } + + @Test + public void testVectorReshapeDenseCP() { + runVectorReshape(false, ExecType.CP); + } + + @Test + public void testVectorReshapeSparseCP() { + runVectorReshape(true, ExecType.CP); + } + + @Test + public void testVectorReshapeDenseMR() { + runVectorReshape(false, ExecType.MR); + } + + @Test + public void testVectorReshapeSparseMR() { + runVectorReshape(true, ExecType.MR); + } + + @Test + public void testVectorReshapeDenseSpark() { + runVectorReshape(false, ExecType.SPARK); + } + + @Test + public void testVectorReshapeSparseSpark() { + runVectorReshape(true, ExecType.SPARK); + } + + private void runVectorReshape(boolean sparse, ExecType et) + { + //rtplatform for MR + RUNTIME_PLATFORM platformOld = rtplatform; + switch( et ){ + case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break; + case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break; + default: rtplatform = RUNTIME_PLATFORM.HYBRID; break; + } + boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; + if( rtplatform == RUNTIME_PLATFORM.SPARK ) + DMLScript.USE_LOCAL_SPARK_CONFIG = true; + + try + { + //register test configuration + TestConfiguration config = getTestConfiguration(TEST_NAME); + loadTestConfiguration(config); + + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + programArgs = new String[]{"-args", input("X"), + String.valueOf(rows2), String.valueOf(cols2), output("R") }; + + fullRScriptName = HOME + TEST_NAME + ".R"; + rCmd = "Rscript" + " " + fullRScriptName + " " + + inputDir() + " " + rows2 + " " + cols2 + " " + expectedDir(); + + double sparsity = sparse ? sparsitySparse : sparsityDense; + double[][] X = getRandomMatrix(rows1, cols1, 0, 1, sparsity, 7); + writeInputMatrixWithMTD("X", X, true); + + runTest(true, false, null, -1); + runRScript(true); + + //compare matrices + HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R"); + HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R"); + TestUtils.compareMatrices(dmlfile, rfile, 10e-10, "Stat-DML", "Stat-R"); + } + finally { + //reset platform for additional tests + rtplatform = platformOld; + DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; + } + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/8707ff1d/src/test/scripts/functions/reorg/VectorReshape.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/reorg/VectorReshape.R b/src/test/scripts/functions/reorg/VectorReshape.R new file mode 100644 index 0000000..6ac63f1 --- /dev/null +++ b/src/test/scripts/functions/reorg/VectorReshape.R @@ -0,0 +1,29 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +args <- commandArgs(TRUE) +options(digits=22) +library("Matrix") + +X=readMM(paste(args[1], "X.mtx", sep="")); +R = matrix(X, as.integer(args[2]), as.integer(args[3]), byrow=TRUE); +writeMM(as(R,"CsparseMatrix"), paste(args[4], "R", sep="")); http://git-wip-us.apache.org/repos/asf/systemml/blob/8707ff1d/src/test/scripts/functions/reorg/VectorReshape.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/reorg/VectorReshape.dml b/src/test/scripts/functions/reorg/VectorReshape.dml new file mode 100644 index 0000000..9c6cab8 --- /dev/null +++ b/src/test/scripts/functions/reorg/VectorReshape.dml @@ -0,0 +1,25 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +X = read($1); +R = matrix(X, rows=$2, cols=$3, byrow=TRUE); +write(R, $4); http://git-wip-us.apache.org/repos/asf/systemml/blob/8707ff1d/src/test_suites/java/org/apache/sysml/test/integration/functions/reorg/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/reorg/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/reorg/ZPackageSuite.java index ea3b6e7..6ad350a 100644 --- a/src/test_suites/java/org/apache/sysml/test/integration/functions/reorg/ZPackageSuite.java +++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/reorg/ZPackageSuite.java @@ -30,7 +30,8 @@ import org.junit.runners.Suite; FullOrderTest.class, FullReverseTest.class, FullTransposeTest.class, - MatrixReshapeTest.class + MatrixReshapeTest.class, + VectorReshapeTest.class, })
