[SYSTEMML-2441] Fix spark matrix market reblock (incorrect output cell) This patch fixes an correctness issues of the spark matrix market reblock, which did not properly ignore the first uncommented row which constitutes meta data of nrow, ncol, and nnz. Hence, the binary output had an incorrect value (of the given nnz) in the bottom-right cell.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/ebe9e8a6 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/ebe9e8a6 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/ebe9e8a6 Branch: refs/heads/master Commit: ebe9e8a6e26139fa48689f48d81aa620abf8f02f Parents: f2a413a Author: Matthias Boehm <[email protected]> Authored: Fri Jul 13 19:17:10 2018 -0700 Committer: Matthias Boehm <[email protected]> Committed: Fri Jul 13 19:17:10 2018 -0700 ---------------------------------------------------------------------- .../instructions/spark/utils/RDDConverterUtils.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/ebe9e8a6/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java index fe5aab1..90d94d0 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java @@ -510,13 +510,20 @@ public class RDDConverterUtils ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); ReblockBuffer rbuff = new ReblockBuffer(_bufflen, _rlen, _clen, _brlen, _bclen); FastStringTokenizer st = new FastStringTokenizer(' '); + boolean first = false; - while( arg0.hasNext() ) - { - //get input string (ignore matrix market comments) + while( arg0.hasNext() ) { + //get input string (ignore matrix market comments as well as + //first row which indicates meta data, i.e., <nrow> <ncol> <nnz>) String strVal = arg0.next().toString(); - if( strVal.startsWith("%") ) + if( strVal.startsWith("%") ) { + first = true; continue; + } + else if (first) { + first = false; + continue; + } //parse input ijv triple st.reset( strVal );
