Repository: systemml Updated Branches: refs/heads/master 11d119877 -> 682da9cdc
[SYSTEMML-2434] Support for matrix market fields real/integer/pattern This patch introduces support for matrix market field types beyond real (i.e., floating point values) in CP and Spark execution modes. Specifically, we now also support integer and pattern types, where the latter only represents the non-zero structure (i.e., boolean values) which is very common for sparse graph datasets. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/682da9cd Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/682da9cd Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/682da9cd Branch: refs/heads/master Commit: 682da9cdcaf30505e0cea6f418271661766c2022 Parents: 11d1198 Author: Matthias Boehm <[email protected]> Authored: Fri Jul 13 22:55:30 2018 -0700 Committer: Matthias Boehm <[email protected]> Committed: Fri Jul 13 22:55:30 2018 -0700 ---------------------------------------------------------------------- .../spark/utils/RDDConverterUtils.java | 7 +- .../runtime/io/FileFormatPropertiesMM.java | 8 + .../sysml/runtime/io/IOUtilFunctions.java | 3 +- .../apache/sysml/runtime/io/ReaderTextCell.java | 88 +++++----- .../runtime/io/ReaderTextCellParallel.java | 79 ++++----- .../apache/sysml/runtime/matrix/data/IJV.java | 3 +- .../runtime/matrix/mapred/ReblockBuffer.java | 3 +- .../sysml/runtime/util/FastStringTokenizer.java | 171 ++++++------------- .../functions/data/MatrixMarketFormatTest.java | 64 +------ 9 files changed, 143 insertions(+), 283 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java index 94dfb55..4f9e5f9 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java @@ -500,7 +500,6 @@ public class RDDConverterUtils { private static final long serialVersionUID = 4907483236186747224L; - @SuppressWarnings("unused") private final FileFormatPropertiesMM _mmProps; protected TextToBinaryBlockFunction(MatrixCharacteristics mc, FileFormatPropertiesMM mmProps) { @@ -531,11 +530,11 @@ public class RDDConverterUtils } //parse input ijv triple - st.reset( strVal ); + st.reset( strVal.toString() ); //reinit tokenizer long row = st.nextLong(); long col = st.nextLong(); - if( row == 0 || col == 0 ) continue; - double val = st.nextDouble(); + double val = (_mmProps == null) ? st.nextDouble() : + _mmProps.isPatternField() ? 1 : _mmProps.isIntField() ? st.nextLong() : st.nextDouble(); //flush buffer if necessary if( rbuff.getSize() >= rbuff.getCapacity() ) http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/main/java/org/apache/sysml/runtime/io/FileFormatPropertiesMM.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/FileFormatPropertiesMM.java b/src/main/java/org/apache/sysml/runtime/io/FileFormatPropertiesMM.java index 0cdbe7c..dd806d4 100644 --- a/src/main/java/org/apache/sysml/runtime/io/FileFormatPropertiesMM.java +++ b/src/main/java/org/apache/sysml/runtime/io/FileFormatPropertiesMM.java @@ -87,6 +87,14 @@ public class FileFormatPropertiesMM extends FileFormatProperties implements Seri public MMSymmetry getSymmetry() { return _symmetry; } + + public boolean isIntField() { + return _field == MMField.INTEGER; + } + + public boolean isPatternField() { + return _field == MMField.PATTERN; + } public static FileFormatPropertiesMM parse(String header) { //example: %%MatrixMarket matrix coordinate real general http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java index e158f61..94941a1 100644 --- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java +++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java @@ -131,8 +131,7 @@ public class IOUtilFunctions } } - public static double parseDoubleParallel( String str ) - { + public static double parseDoubleParallel( String str ) { //return FloatingDecimal.parseDouble(str); return Double.parseDouble(str); } http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java index 321aec4..45f117f 100644 --- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java +++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java @@ -38,6 +38,7 @@ import org.apache.hadoop.mapred.TextInputFormat; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.matrix.data.DenseBlock; +import org.apache.sysml.runtime.matrix.data.IJV; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.util.FastStringTokenizer; @@ -47,7 +48,7 @@ public class ReaderTextCell extends MatrixReader { protected final boolean _allowRawRead; protected final boolean _isMMFile; - protected FileFormatProperties _mmProps = null; + protected FileFormatPropertiesMM _mmProps = null; public ReaderTextCell(InputInfo info) { this(info, true); @@ -122,7 +123,7 @@ public class ReaderTextCell extends MatrixReader LongWritable key = new LongWritable(); Text value = new Text(); - int row = -1, col = -1; + IJV cell = new IJV(); long nnz = 0; try @@ -132,30 +133,22 @@ public class ReaderTextCell extends MatrixReader for(InputSplit split: splits) { RecordReader<LongWritable,Text> reader = informat.getRecordReader(split, job, Reporter.NULL); try { - if( sparse ) //SPARSE<-value - { + if( sparse ) { //SPARSE<-value while( reader.next(key, value) ) { - st.reset( value.toString() ); //reinit tokenizer - row = st.nextInt() - 1; - col = st.nextInt() - 1; - if(row == -1 || col == -1) continue; - double lvalue = st.nextDouble(); - dest.appendValue(row, col, lvalue); + cell = parseCell(value.toString(), st, cell, _mmProps); + if( cell.getV() != 0 ) + dest.appendValue(cell.getI(), cell.getJ(), cell.getV()); } - dest.sortSparseRows(); } - else //DENSE<-value - { + else { //DENSE<-value DenseBlock a = dest.getDenseBlock(); while( reader.next(key, value) ) { - st.reset( value.toString() ); //reinit tokenizer - row = st.nextInt()-1; - col = st.nextInt()-1; - if(row == -1 || col == -1) continue; - double lvalue = st.nextDouble(); - a.set( row, col, lvalue ); - nnz += (lvalue != 0) ? 1 : 0; + cell = parseCell(value.toString(), st, cell, _mmProps); + if( cell.getV() != 0 ) { + a.set( cell.getI(), cell.getJ(), cell.getV() ); + nnz ++; + } } } } @@ -169,13 +162,22 @@ public class ReaderTextCell extends MatrixReader } catch(Exception ex) { //post-mortem error handling and bounds checking - if( row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen ) - throw new IOException("Matrix cell ["+(row+1)+","+(col+1)+"] " + - "out of overall matrix range [1:"+rlen+",1:"+clen+"]."); + if( cell.getI() < 0 || cell.getI() + 1 > rlen || cell.getJ() < 0 || cell.getJ() + 1 > clen ) + throw new IOException("Matrix cell ["+(cell.getI()+1)+","+(cell.getJ()+1)+"] " + + "out of overall matrix range [1:"+rlen+",1:"+clen+"]."); else throw new IOException( "Unable to read matrix in text cell format.", ex ); } } + + protected static IJV parseCell(String line, FastStringTokenizer st, IJV cell, FileFormatPropertiesMM mmProps) { + st.reset( line.toString() ); //reinit tokenizer + int row = st.nextInt() - 1; + int col = st.nextInt() - 1; + double value = (mmProps == null) ? st.nextDouble() : + mmProps.isPatternField() ? 1 : mmProps.isIntField() ? st.nextLong() : st.nextDouble(); + return cell.set(row, col, value); + } private static void readRawTextCellMatrixFromHDFS( Path path, JobConf job, FileSystem fs, MatrixBlock dest, long rlen, long clen, int brlen, int bclen, boolean matrixMarket ) throws IOException @@ -191,12 +193,11 @@ public class ReaderTextCell extends MatrixReader throws IOException { BufferedReader br = new BufferedReader(new InputStreamReader( is )); - @SuppressWarnings("unused") FileFormatPropertiesMM mmProps = null; boolean sparse = dest.isInSparseFormat(); String value = null; - int row = -1, col = -1; + IJV cell = new IJV(); long nnz = 0; // Read the header lines, if reading from a matrixMarket file @@ -226,40 +227,31 @@ public class ReaderTextCell extends MatrixReader { FastStringTokenizer st = new FastStringTokenizer(' '); - if( sparse ) //SPARSE<-value - { - while( (value=br.readLine())!=null ) - { - st.reset( value ); //reinit tokenizer - row = st.nextInt()-1; - col = st.nextInt()-1; - if(row == -1 || col == -1) continue; - double lvalue = st.nextDouble(); - dest.appendValue(row, col, lvalue); + if( sparse ) { //SPARSE<-value + while( (value=br.readLine())!=null ) { + cell = parseCell(value.toString(), st, cell, mmProps); + if( cell.getV() != 0 ) + dest.appendValue(cell.getI(), cell.getJ(), cell.getV()); } - dest.sortSparseRows(); } - else //DENSE<-value - { + else { //DENSE<-value DenseBlock a = dest.getDenseBlock(); while( (value=br.readLine())!=null ) { - st.reset( value ); //reinit tokenizer - row = st.nextInt()-1; - col = st.nextInt()-1; - if(row == -1 || col == -1) continue; - double lvalue = st.nextDouble(); - a.set( row, col, lvalue ); - nnz += (lvalue != 0) ? 1 : 0; + cell = parseCell(value.toString(), st, cell, mmProps); + if( cell.getV() != 0 ) { + a.set( cell.getI(), cell.getJ(), cell.getV() ); + nnz ++; + } } dest.setNonZeros(nnz); } } catch(Exception ex) { //post-mortem error handling and bounds checking - if( row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen ) - throw new IOException("Matrix cell ["+(row+1)+","+(col+1)+"] " + - "out of overall matrix range [1:"+rlen+",1:"+clen+"].", ex); + if( cell.getI() < 0 || cell.getI() + 1 > rlen || cell.getJ() < 0 || cell.getJ() + 1 > clen ) + throw new IOException("Matrix cell ["+(cell.getI()+1)+","+(cell.getJ()+1)+"] " + + "out of overall matrix range [1:"+rlen+",1:"+clen+"].", ex); else throw new IOException( "Unable to read matrix in raw text cell format.", ex ); } http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java index 3a67e56..1ce0164 100644 --- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java +++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java @@ -37,6 +37,7 @@ import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.runtime.matrix.data.DenseBlock; +import org.apache.sysml.runtime.matrix.data.IJV; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.util.CommonThreadPool; @@ -95,7 +96,7 @@ public class ReaderTextCellParallel extends ReaderTextCell InputSplit[] splits = informat.getSplits(job, par); ArrayList<ReadTask> tasks = new ArrayList<>(); for( InputSplit split : splits ){ - ReadTask t = new ReadTask(split, informat, job, dest, rlen, clen, _isMMFile); + ReadTask t = new ReadTask(split, informat, job, dest, rlen, clen, _isMMFile, _mmProps); tasks.add(t); } @@ -106,7 +107,7 @@ public class ReaderTextCellParallel extends ReaderTextCell long lnnz = 0; for( Future<Long> task : rt ) lnnz += task.get(); - + //post-processing dest.setNonZeros( lnnz ); if( dest.isInSparseFormat() ) @@ -121,17 +122,17 @@ public class ReaderTextCellParallel extends ReaderTextCell public static class ReadTask implements Callable<Long> { - private InputSplit _split = null; - private boolean _sparse = false; - private TextInputFormat _informat = null; - private JobConf _job = null; - private MatrixBlock _dest = null; - private long _rlen = -1; - private long _clen = -1; - private boolean _matrixMarket = false; + private final InputSplit _split; + private final boolean _sparse; + private final TextInputFormat _informat; + private final JobConf _job; + private final MatrixBlock _dest; + private final long _rlen; + private final long _clen; + private final boolean _matrixMarket; + private final FileFormatPropertiesMM _mmProps; - public ReadTask( InputSplit split, TextInputFormat informat, JobConf job, MatrixBlock dest, long rlen, long clen, boolean matrixMarket ) - { + public ReadTask( InputSplit split, TextInputFormat informat, JobConf job, MatrixBlock dest, long rlen, long clen, boolean mm, FileFormatPropertiesMM mmProps ) { _split = split; _sparse = dest.isInSparseFormat(); _informat = informat; @@ -139,7 +140,8 @@ public class ReaderTextCellParallel extends ReaderTextCell _dest = dest; _rlen = rlen; _clen = clen; - _matrixMarket = matrixMarket; + _matrixMarket = mm; + _mmProps = mmProps; } @Override @@ -150,10 +152,7 @@ public class ReaderTextCellParallel extends ReaderTextCell //writables for reuse during read LongWritable key = new LongWritable(); Text value = new Text(); - - //required for error handling - int row = -1; - int col = -1; + IJV cell = new IJV(); FastStringTokenizer st = new FastStringTokenizer(' '); RecordReader<LongWritable,Text> reader = _informat.getRecordReader(_split, _job, Reporter.NULL); @@ -171,63 +170,47 @@ public class ReaderTextCellParallel extends ReaderTextCell //process current value (otherwise ignore following meta data) if( !foundComment ) { - st.reset( value.toString() ); //reinit tokenizer - row = st.nextInt()-1; - col = st.nextInt()-1; - if(row != -1 || col != -1) { - double lvalue = st.nextDoubleForParallel(); + cell = parseCell(value.toString(), st, cell, _mmProps); + if( cell.getV() != 0 ) { synchronized( _dest ){ //sparse requires lock - _dest.appendValue(row, col, lvalue); + _dest.appendValue(cell.getI(), cell.getJ(), cell.getV()); lnnz++; } } } } - if( _sparse ) //SPARSE<-value - { + if( _sparse ) { //SPARSE<-value CellBuffer buff = new CellBuffer(); - while( reader.next(key, value) ) { - st.reset( value.toString() ); //reinit tokenizer - row = st.nextInt() - 1; - col = st.nextInt() - 1; - if(row == -1 || col == -1) continue; - double lvalue = st.nextDoubleForParallel(); - - buff.addCell(row, col, lvalue); - //capacity buffer flush on demand + buff.addCell(parseCell(value.toString(), st, cell, _mmProps)); if( buff.size()>=CellBuffer.CAPACITY ) synchronized( _dest ){ //sparse requires lock lnnz += buff.size(); buff.flushCellBufferToMatrixBlock(_dest); } } - //final buffer flush synchronized( _dest ){ //sparse requires lock lnnz += buff.size(); buff.flushCellBufferToMatrixBlock(_dest); } } - else //DENSE<-value - { + else { //DENSE<-value DenseBlock a = _dest.getDenseBlock(); while( reader.next(key, value) ) { - st.reset( value.toString() ); //reinit tokenizer - row = st.nextInt()-1; - col = st.nextInt()-1; - if(row == -1 || col == -1) continue; - double lvalue = st.nextDoubleForParallel(); - a.set( row, col, lvalue ); - lnnz += (lvalue!=0) ? 1 : 0; + cell = parseCell(value.toString(), st, cell, _mmProps); + if( cell.getV() != 0 ) { + a.set( cell.getI(), cell.getJ(), cell.getV() ); + lnnz ++; + } } } } catch(Exception ex) { //post-mortem error handling and bounds checking - if( row < 0 || row + 1 > _rlen || col < 0 || col + 1 > _clen ) - throw new RuntimeException("Matrix cell ["+(row+1)+","+(col+1)+"] " + + if( cell.getI() < 0 || cell.getI() + 1 > _rlen || cell.getJ() < 0 || cell.getJ() + 1 > _clen ) + throw new RuntimeException("Matrix cell ["+(cell.getI()+1)+","+(cell.getJ()+1)+"] " + "out of overall matrix range [1:"+_rlen+",1:"+_clen+"]. ", ex); else throw new RuntimeException("Unable to read matrix in text cell format. ", ex); @@ -261,6 +244,10 @@ public class ReaderTextCellParallel extends ReaderTextCell _pos = -1; } + public void addCell(IJV cell) { + addCell(cell.getI(), cell.getJ(), cell.getV()); + } + public void addCell(int rlen, int clen, double val) { if( val==0 ) return; _pos++; http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java b/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java index f4c6c7a..86c25e3 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java @@ -33,10 +33,11 @@ public class IJV //do nothing } - public void set(int i, int j, double v) { + public IJV set(int i, int j, double v) { _i = i; _j = j; _v = v; + return this; } public int getI() { http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java index 8d6f2e6..7fc24ed 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java @@ -71,8 +71,7 @@ public class ReblockBuffer _bclen = bclen; } - public void appendCell( long r, long c, double v ) - { + public void appendCell( long r, long c, double v ) { long tmp = Double.doubleToRawLongBits(v); _buff[_count][0] = r; _buff[_count][1] = c; http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/main/java/org/apache/sysml/runtime/util/FastStringTokenizer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/util/FastStringTokenizer.java b/src/main/java/org/apache/sysml/runtime/util/FastStringTokenizer.java index 2320fa1..fbc380e 100644 --- a/src/main/java/org/apache/sysml/runtime/util/FastStringTokenizer.java +++ b/src/main/java/org/apache/sysml/runtime/util/FastStringTokenizer.java @@ -32,125 +32,60 @@ public class FastStringTokenizer implements Serializable { private static final long serialVersionUID = 4051436015710778611L; private String _string = null; - private char _del = 0; - private int _pos = -1; + private char _del = 0; + private int _pos = -1; - /** - * Constructs a new StringTokenizer for string using the specified - * delimiter - * - * @param delimiter - * the delimiter to use - */ - public FastStringTokenizer(char delimiter) - { - _del = delimiter; - reset( null ); - } + /** + * Constructs a new StringTokenizer for string using the specified + * delimiter + * + * @param delimiter + * the delimiter to use + */ + public FastStringTokenizer(char delimiter) { + _del = delimiter; + reset( null ); + } - public void reset( String string ) - { - _string = string; - _pos = 0; - } - - /** - * Returns the next token in the string as a String. - * - * @return next token in the string as a String - */ - public String nextToken() - { - int len = _string.length(); - int start = _pos; - - //find start (skip over leading delimiters) - while(start < len && _del == _string.charAt(start) ) - start++; - - //find end (next delimiter) and return - if(start < len) { - _pos = _string.indexOf(_del, start); - if( start < _pos && _pos < len ) - return _string.substring(start, _pos); - else - return _string.substring(start); - } - - //no next token + public void reset( String string ) { + _string = string; + _pos = 0; + } + + /** + * Returns the next token in the string as a String. + * + * @return next token in the string as a String + */ + public String nextToken() { + int len = _string.length(); + int start = _pos; + + //find start (skip over leading delimiters) + while(start < len && _del == _string.charAt(start) ) + start++; + + //find end (next delimiter) and return + if(start < len) { + _pos = _string.indexOf(_del, start); + if( start < _pos && _pos < len ) + return _string.substring(start, _pos); + else + return _string.substring(start); + } + //no next token throw new NoSuchElementException(); - } - - //////////////////////////////////////// - // Custom parsing methods for textcell - //////////////////////////////////////// - - public int nextInt() - { - return Integer.parseInt( nextToken() ); - } - - public long nextLong() - { - return Long.parseLong( nextToken() ); - } - - public double nextDouble() - { - return Double.parseDouble( nextToken() ); - - //see nextDoubleForParallel, we use the same double parsing - //for sequential and parallel parsing because (1) it is faster (~10%) - //and (2) for consistency between sequential and parallel readers - - //return FloatingDecimal.parseDouble(nextToken()); - } - - public double nextDoubleForParallel() - { - //JDK 8 floating decimal, which removes a severe scalability bottleneck - //(synchronized static cache) in JDK7 - //return FloatingDecimal.parseDouble(nextToken()); - return Double.parseDouble( nextToken() ); - - /* - //return Double.parseDouble( nextToken() ); - - //NOTE: Depending on the platform string-2-double conversions were - //the main bottleneck in reading text data. Furthermore, we observed - //severe contention on multi-threaded parsing on Linux JDK. - // --- - //This is a known issue and has been fixed in JDK8. - //JDK-7032154 : Performance tuning of sun.misc.FloatingDecimal/FormattedFloatingDecimal - - // Simple workaround without JDK8 code, however, this does NOT guarantee exactly - // the same result due to potential for round off errors. - - String val = nextToken(); - double ret = 0; - - if( UtilFunctions.isSimpleDoubleNumber(val) ) - { - int ix = val.indexOf('.'); - if( ix > 0 ) //DOUBLE parsing - { - String s1 = val.substring(0, ix); - String s2 = val.substring(ix+1); - long tmp1 = Long.parseLong(s1); - long tmp2 = Long.parseLong(s2); - ret = (double)tmp2 / Math.pow(10, s2.length()) + tmp1; - } - else //LONG parsing and cast to double - ret = (double)Long.parseLong(val); - } - else - { - //fall-back to slow default impl if special characters - //e.g., ...E-0X, NAN, +-INFINITY, etc - ret = Double.parseDouble( val ); - } - - return ret; - */ - } + } + + public int nextInt() { + return Integer.parseInt( nextToken() ); + } + + public long nextLong() { + return Long.parseLong( nextToken() ); + } + + public double nextDouble() { + return Double.parseDouble( nextToken() ); + } } http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java index 3ec542e..59e3aa5 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java @@ -84,11 +84,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase } @Test - public void testMMCooRealSymmetricMR() { - runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.SYMMETRIC, ExecType.MR); - } - - @Test public void testMMCooRealSkewSymmetricCP() { runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.SKEW_SYMMETRIC, ExecType.CP); } @@ -97,11 +92,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase public void testMMCooRealSkewSymmetricSp() { runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.SKEW_SYMMETRIC, ExecType.SPARK); } - - @Test - public void testMMCooRealSkewSymmetricMR() { - runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.SKEW_SYMMETRIC, ExecType.MR); - } @Test public void testMMCooIntegerGeneralCP() { @@ -129,11 +119,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase } @Test - public void testMMCooIntegerSymmetricMR() { - runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.SYMMETRIC, ExecType.MR); - } - - @Test public void testMMCooIntegerSkewSymmetricCP() { runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.SKEW_SYMMETRIC, ExecType.CP); } @@ -142,11 +127,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase public void testMMCooIntegerSkewSymmetricSp() { runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.SKEW_SYMMETRIC, ExecType.SPARK); } - - @Test - public void testMMCooIntegerSkewSymmetricMR() { - runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.SKEW_SYMMETRIC, ExecType.MR); - } @Test public void testMMCooPatternGeneralCP() { @@ -157,11 +137,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase public void testMMCooPatternGeneralSp() { runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.PATTERN, MMSymmetry.GENERAL, ExecType.SPARK); } - - @Test - public void testMMCooPatternGeneralMR() { - runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.PATTERN, MMSymmetry.GENERAL, ExecType.MR); - } @Test public void testMMCooPatternSymmetricCP() { @@ -174,11 +149,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase } @Test - public void testMMCooPatternSymmetricMR() { - runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.PATTERN, MMSymmetry.SYMMETRIC, ExecType.MR); - } - - @Test public void testMMArrRealGeneralCP() { runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.GENERAL, ExecType.CP); } @@ -187,11 +157,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase public void testMMArrRealGeneralSp() { runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.GENERAL, ExecType.SPARK); } - - @Test - public void testMMArrRealGeneralMR() { - runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.GENERAL, ExecType.MR); - } @Test public void testMMArrRealSymmetricCP() { @@ -202,11 +167,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase public void testMMArrRealSymmetricSp() { runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.SYMMETRIC, ExecType.SPARK); } - - @Test - public void testMMArrRealSymmetricMR() { - runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.SYMMETRIC, ExecType.MR); - } @Test public void testMMArrRealSkewSymmetricCP() { @@ -217,11 +177,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase public void testMMArrRealSkewSymmetricSp() { runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.SKEW_SYMMETRIC, ExecType.SPARK); } - - @Test - public void testMMArrRealSkewSymmetricMR() { - runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.SKEW_SYMMETRIC, ExecType.MR); - } @Test public void testMMArrIntegerGeneralCP() { @@ -232,11 +187,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase public void testMMArrIntegerGeneralSp() { runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.GENERAL, ExecType.SPARK); } - - @Test - public void testMMArrIntegerGeneralMR() { - runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.GENERAL, ExecType.MR); - } @Test public void testMMArrIntegerSymmetricCP() { @@ -247,11 +197,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase public void testMMArrIntegerSymmetricSp() { runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.SYMMETRIC, ExecType.SPARK); } - - @Test - public void testMMArrIntegerSymmetricMR() { - runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.SYMMETRIC, ExecType.MR); - } @Test public void testMMArrIntegerSkewSymmetricCP() { @@ -263,11 +208,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.SKEW_SYMMETRIC, ExecType.SPARK); } - @Test - public void testMMArrIntegerSkewSymmetricMR() { - runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.SKEW_SYMMETRIC, ExecType.MR); - } - private void runMatrixMarketFormatTest(MMFormat fmt, MMField field, MMSymmetry symmetry, ExecType et) { //rtplatform for MR @@ -360,8 +300,8 @@ public class MatrixMarketFormatTest extends AutomatedTestBase sb.append(cell.getJ()+1); if( field != MMField.PATTERN ) { sb.append(' '); - sb.append(String.valueOf((field == MMField.INTEGER) ? - (int) cell.getV() : cell.getV())); + sb.append((field == MMField.INTEGER) ? + String.valueOf((int) cell.getV()) : String.valueOf(cell.getV())); } sb.append('\n'); br.write( sb.toString() ); //same as append
